diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9f28ec942..3d6f1daf8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -37,6 +37,10 @@ content/includes/nginxaas-azure/* @nginx/n4a-docs-approvers content/ngf/* @nginx/nginx-gateway-fabric content/includes/ngf/* @nginx/nginx-gateway-fabric +# NGINX Ingress Controller +content/nic/* @nginx/kic +content/includes/nic/* @nginx/kic + # NGINX Instance Manager content/nms/nim/* @nginx/nim-docs-approvers content/nim/* @nginx/nim-docs-approvers diff --git a/config/_default/config.toml b/config/_default/config.toml index 10337a908..faf409f46 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -7,18 +7,19 @@ pygmentsUseClasses = true enableGitInfo = true [permalinks] + agent = '/nginx-agent/:sections[1:]/:filename' amplify = '/nginx-amplify/:sections[1:]/:filename' controller = '/nginx-controller/:sections[1:]/:filename' mesh = '/nginx-service-mesh/:sections[1:]/:filename' modsec-waf = '/nginx-waf/:sections[1:]/:filename' nap-dos = '/nginx-app-protect-dos/:sections[1:]/:filename' nap-waf = '/nginx-app-protect-waf/:sections[1:]/:filename' + nginxaas = '/nginxaas/azure/:sections[1:]/:filename' ngf = '/nginx-gateway-fabric/:sections[1:]/:filename' + nic = '/nginx-ingress-controller/:sections[1:]/:filename' nim = '/nginx-instance-manager/:sections[1:]/:filename' nms = '/nginx-management-suite/:sections[1:]/:filename' unit = '/nginx-unit/:sections[1:]/:filename' - agent = '/nginx-agent/:sections[1:]/:filename' - nginxaas = '/nginxaas/azure/:sections[1:]/:filename' [caches] [caches.modules] diff --git a/content/includes/nic/compatibility-tables/nic-nap.md b/content/includes/nic/compatibility-tables/nic-nap.md new file mode 100644 index 000000000..aaf914ca1 --- /dev/null +++ b/content/includes/nic/compatibility-tables/nic-nap.md @@ -0,0 +1,10 @@ +The following table shows compatibility between NGINX Ingress Controller (NIC) and NGINX App Protect WAF (NAP-WAF) versions: + +{{< bootstrap-table "table table-striped table-responsive" >}} +| NIC Version | NAP-WAF Version | Config Manager | Enforcer | +| ------------------- | --------------- | -------------- | -------- | +| {{< nic-version >}} | 34+5.332 | 5.6.0 | 5.6.0 | +| 4.0.1 | 33+5.264 | 5.5.0 | 5.5.0 | +| 3.7.2 | 32+5.1 | 5.3.0 | 5.3.0 | +| 3.6.2 | 32+5.48 | 5.2.0 | 5.2.0 | +{{% /bootstrap-table %}} diff --git a/content/includes/nic/configuration/_index.md b/content/includes/nic/configuration/_index.md new file mode 100644 index 000000000..7e1500973 --- /dev/null +++ b/content/includes/nic/configuration/_index.md @@ -0,0 +1,8 @@ +--- +title: Configuration +description: +weight: 1400 +menu: + docs: + parent: NGINX Ingress Controller +--- diff --git a/content/includes/nic/configuration/access-control.md b/content/includes/nic/configuration/access-control.md new file mode 100644 index 000000000..fade2941c --- /dev/null +++ b/content/includes/nic/configuration/access-control.md @@ -0,0 +1,120 @@ +--- +title: Deploy a Policy for access control +weight: 900 +toc: true +docs: DOCS-000 +--- + +This topic describes how to use F5 NGINX Ingress Controller to apply and update a Policy for access control. It demonstrates it using an example application and a [VirtualServer custom resource]({{< ref "/configuration/virtualserver-and-virtualserverroute-resources.md" >}}). + +--- + +## Before you begin + +You should have a [working NGINX Ingress Controller]({{< ref "/installation/installing-nic/installation-with-helm.md" >}}) instance. + +For ease of use in shell commands, set two shell variables: + +1. The public IP address for your NGINX Ingress Controller instance. + +```shell +IC_IP= +``` + +2. The HTTP port of the same instance. + +```shell +IC_HTTP_PORT= +``` + +--- + +## Deploy the example application + +Create the file _webapp.yaml_ with the following contents: + +{{< ghcode "https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/custom-resources/access-control/webapp.yaml" >}} + +Apply it using `kubectl`: + +```shell +kubectl apply -f webapp.yaml +``` + +--- + +## Deploy a Policy to create a deny rule + +Create a file named _access-control-policy-deny.yaml_. The highlighted _deny_ field will be used by the example application, and should be changed to the subnet of your machine. + +{{< ghcode "https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/custom-resources/access-control/access-control-policy-deny.yaml" "hl_lines=7-8" >}} + +Apply the policy: + +```shell +kubectl apply -f access-control-policy-deny.yaml +``` + +--- + +## Configure load balancing + +Create a file named _virtual-server.yaml_ for the VirtualServer resource. The _policies_ field references the access control Policy created in the previous section. + +{{< ghcode "https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/custom-resources/access-control/virtual-server.yaml" "hl_lines=7-8" >}} + +Apply the policy: + +```shell +kubectl apply -f virtual-server.yaml +``` + +--- + +## Test the example application + +Use `curl` to attempt to access the application: + +```shell +curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP http://webapp.example.com:$IC_HTTP_PORT +``` +```text + +403 Forbidden + +

403 Forbidden

+ + +``` + +The *403* response is expected, successfully blocking your machine. + +--- + +## Update the Policy to create an allow rule + +Update the Policy with the file _access-control-policy-allow.yaml_, setting the _allow_ field to the subnet of your machine. + +{{< ghcode "https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/custom-resources/access-control/access-control-policy-allow.yaml" "hl_lines=7-8" >}} + +Apply the Policy: + +```shell +kubectl apply -f access-control-policy-allow.yaml +``` + +---- + +## Verify the Policy update + +Attempt to access the application again: + +```shell +curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP http://webapp.example.com:$IC_HTTP_PORT +``` +```text +Server address: 10.64.0.13:8080 +Server name: webapp-5cbbc7bd78-wf85w +``` + +The successful response demonstrates that the policy has been updated. diff --git a/content/includes/nic/configuration/configuration-examples.md b/content/includes/nic/configuration/configuration-examples.md new file mode 100644 index 000000000..0a0e6e3bb --- /dev/null +++ b/content/includes/nic/configuration/configuration-examples.md @@ -0,0 +1,13 @@ +--- +docs: DOCS-584 +doctypes: +- '' +title: Configuration examples +toc: true +weight: 400 +--- + +Our [GitHub repo](https://github.com/nginx/kubernetes-ingress) includes a number of configuration examples: + +- [*Examples of Custom Resources*](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources) show how to advanced NGINX features by using VirtualServer, VirtualServerRoute, TransportServer and Policy Custom Resources. +- [*Examples of Ingress Resources*](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources) show how to use advanced NGINX features in Ingress resources with annotations. diff --git a/content/includes/nic/configuration/global-configuration/_index.md b/content/includes/nic/configuration/global-configuration/_index.md new file mode 100644 index 000000000..4f71e2f5f --- /dev/null +++ b/content/includes/nic/configuration/global-configuration/_index.md @@ -0,0 +1,8 @@ +--- +title: Global configuration +description: +weight: 100 +menu: + docs: + parent: NGINX Ingress Controller +--- diff --git a/content/includes/nic/configuration/global-configuration/command-line-arguments.md b/content/includes/nic/configuration/global-configuration/command-line-arguments.md new file mode 100644 index 000000000..230345052 --- /dev/null +++ b/content/includes/nic/configuration/global-configuration/command-line-arguments.md @@ -0,0 +1,685 @@ +--- +docs: DOCS-585 +doctypes: +- '' +title: Command-line arguments +toc: true +weight: 100 +--- + +F5 NGINX Ingress Controller supports several command-line arguments, which are set based on installation method: + +- If you're using *Kubernetes Manifests* to install NGINX Ingress Controller, modify the Manifests to set the command-line arguments. View the [Installation with Manifests]({{}}) topic for more information. +- If you're using *Helm* to install NGINX Ingress Controller, modify the parameters of the Helm chart to set the command-line arguments. View the [Installation with Helm]({{}}) topic for more information. + + + +### -enable-snippets + +Enable custom NGINX configuration snippets in Ingress, VirtualServer, VirtualServerRoute and TransportServer resources. + +Default `false`. + + + +--- + +### -default-server-tls-secret `` + +Secret with a TLS certificate and key for TLS termination of the default server. + +- If not set, certificate and key in the file `/etc/nginx/secrets/default` are used. +- If `/etc/nginx/secrets/default` doesn't exist, NGINX Ingress Controller will configure NGINX to reject TLS connections to the default server. +- If a secret is set, but NGINX Ingress Controller is not able to fetch it from Kubernetes API, or it is not set and NGINX Ingress Controller fails to read the file "/etc/nginx/secrets/default", NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -wildcard-tls-secret `` + +A Secret with a TLS certificate and key for TLS termination of every Ingress/VirtualServer host for which TLS termination is enabled but the Secret is not specified. + +- If the argument is not set, for such Ingress/VirtualServer hosts NGINX will break any attempt to establish a TLS connection +- If the argument is set, but NGINX Ingress Controller is not able to fetch the Secret from Kubernetes API, NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -enable-custom-resources + +Enables custom resources. + +Default `true`. + + + + +--- + +### -enable-oidc + +Enables OIDC policies. + +Default `false`. + + + +--- + +### -enable-leader-election + +Enables Leader election to avoid multiple replicas of the controller reporting the status of Ingress, VirtualServer and VirtualServerRoute resources -- only one replica will report status. +Default `true`. + +See [-report-ingress-status](#cmdoption-report-ingress-status) flag. + + + +--- + +### -enable-tls-passthrough + +Enable TLS Passthrough on port 443. + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + + +--- + +### -tls-passthrough-port `` + +Set the port for TLS Passthrough. +Format: `[1024 - 65535]` (default `443`) + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + + +--- + +### -enable-cert-manager + +Enable x509 automated certificate management for VirtualServer resources using cert-manager (cert-manager.io). + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + + +--- + +### -enable-external-dns + +Enable integration with ExternalDNS for configuring public DNS entries for VirtualServer resources using [ExternalDNS](https://github.com/kubernetes-sigs/external-dns). + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + +--- + +### -external-service `` + +Specifies the name of the service with the type LoadBalancer through which the NGINX Ingress Controller pods are exposed externally. The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. + +For Ingress resources only: Requires [-report-ingress-status](#cmdoption-report-ingress-status). + + + +--- + +### -ingresslink `` + +Specifies the name of the IngressLink resource, which exposes the NGINX Ingress Controller pods via a BIG-IP system. The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. + +For Ingress resources only: Requires [-report-ingress-status](#cmdoption-report-ingress-status). + + + +--- + +### -global-configuration `` + +A GlobalConfiguration resource for global configuration of NGINX Ingress Controller. + +Format: `/` + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + + +--- + +### -health-status + +Adds a location "/nginx-health" to the default server. The location responds with the 200 status code for any request. + +Useful for external health-checking of NGINX Ingress Controller. + + + +--- + +### -health-status-uri `` + +Sets the URI of health status location in the default server. Requires [-health-status](#cmdoption-health-status). (default `/nginx-health`) + + + +--- + +### -ingress-class `` + +The `-ingress-class` argument refers to the name of the resource `kind: IngressClass`. An IngressClass resource with a name equal to the class must be deployed. Otherwise, NGINX Ingress Controller will fail to start. +NGINX Ingress Controller will only process Ingress resources that belong to its class (Whose `ingressClassName` value matches the value of `-ingress-class`), skipping the ones without it. It will also process all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the `ingressClassName` field. + +Default `nginx`. + + + +--- + +### -ingress-template-path `` + +Path to the ingress NGINX configuration template for an ingress resource. Default for NGINX is `nginx.ingress.tmpl`; default for NGINX Plus is `nginx-plus.ingress.tmpl`. + + + +--- + +### -leader-election-lock-name `` + +Specifies the name of the ConfigMap, within the same namespace as the controller, used as the lock for leader election. + +Requires [-enable-leader-election](#cmdoption-enable-leader-election). + + + +--- + +### -log_backtrace_at `` + +When logging hits line `file:N`, emit a stack trace. + + + +--- + +### -main-template-path `` + +Path to the main NGINX configuration template. + +- Default for NGINX is `nginx.tmpl`. +- Default for NGINX Plus is `nginx-plus.tmpl`. + + + +--- + +### -nginx-configmaps `` + +A ConfigMap resource for customizing NGINX configuration. If a ConfigMap is set, but NGINX Ingress Controller is not able to fetch it from Kubernetes API, NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -mgmt-configmap `` + +The Management ConfigMap resource is used for customizing the NGINX mgmt block. If using NGINX Plus, a Management ConfigMap must be set. If NGINX Ingress Controller is not able to fetch it from Kubernetes API, NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -nginx-debug + +Enable debugging for NGINX. Uses the nginx-debug binary. Requires 'error-log-level: debug' in the ConfigMap. + + + +--- + +### -nginx-plus + +Enable support for NGINX Plus. + + + +--- + +### -nginx-reload-timeout `` + +Timeout in milliseconds which NGINX Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. + +Default is 60000. + + + +--- + +### -nginx-status + +Enable the NGINX stub_status, or the NGINX Plus API. + +Default `true`. + + + +--- + +### -nginx-status-allow-cidrs `` + +Add IP/CIDR blocks to the allow list for NGINX stub_status or the NGINX Plus API. + +Separate multiple IP/CIDR by commas. (default `127.0.0.1,::1`) + + + +--- + +### -nginx-status-port `` + +Set the port where the NGINX stub_status or the NGINX Plus API is exposed. + +Format: `[1024 - 65535]` (default `8080`) + + + +--- + +### -proxy `` + +{{< warning >}} This argument is intended for testing purposes only. {{< /warning >}} + +Use a proxy server to connect to Kubernetes API started with `kubectl proxy`. + +NGINX Ingress Controller does not start NGINX and does not write any generated NGINX configuration files to disk. + + + +--- + +### -report-ingress-status + +Updates the address field in the status of Ingress resources. + +Requires the [-external-service](#cmdoption-external-service) or [-ingresslink](#cmdoption-ingresslink) flag, or the `external-status-address` key in the ConfigMap. + + + +--- + +### -transportserver-template-path `` + +Path to the TransportServer NGINX configuration template for a TransportServer resource. + +- Default for NGINX is `nginx.transportserver.tmpl`. +- Default for NGINX Plus is `nginx-plus.transportserver.tmpl`. + + + +--- + +### -log-level `` + +Log level for Ingress Controller logs. Allowed values: fatal, error, warn, info, debug, trace. + +- Default is `info`. + + + +--- + +### -log-format `` + +Log format for Ingress Controller logs. Allowed values: glog, json, text. + +- Default is `glog`. + + + +--- + +### -version + +Print the version, git-commit hash and build date and exit. + + + +--- + +### -virtualserver-template-path `` + +Path to the VirtualServer NGINX configuration template for a VirtualServer resource. + +- Default for NGINX is `nginx.virtualserver.tmpl`. +- Default for NGINX Plus is `nginx-plus.virtualserver.tmpl`. + + + + +--- + +### -vmodule `` + +A comma-separated list of pattern=N settings for file-filtered logging. + + + +--- + +### -watch-namespace `` + +Comma separated list of namespaces NGINX Ingress Controller should watch for resources. By default NGINX Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace-label". + + + +--- + +### -watch-namespace-label `` + +Configures NGINX Ingress Controller to watch only those namespaces with label foo=bar. By default NGINX Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace". + + + +--- + +### -watch-secret-namespace `` + +Comma separated list of namespaces NGINX Ingress Controller should watch for secrets. If this arg is not configured, NGINX Ingress Controller watches the same namespaces for all resources, see "watch-namespace" and "watch-namespace-label". All namespaces included with this argument must be part of either `-watch-namespace` or `-watch-namespace-label`. + + + +--- + +### -enable-prometheus-metrics + +Enables exposing NGINX or NGINX Plus metrics in the Prometheus format. + + + +--- + +### -prometheus-metrics-listen-port `` + +Sets the port where the Prometheus metrics are exposed. + +Format: `[1024 - 65535]` (default `9113`) + + + +--- + +### -prometheus-tls-secret `` + +A Secret with a TLS certificate and key for TLS termination of the Prometheus metrics endpoint. + +- If the argument is not set, the Prometheus endpoint will not use a TLS connection. +- If the argument is set, but NGINX Ingress Controller is not able to fetch the Secret from Kubernetes API, NGINX Ingress Controller will fail to start. + + + +--- + +### -enable-service-insight + +Exposes the Service Insight endpoint for Ingress Controller. + + + +--- + +### -service-insight-listen-port `` + +Sets the port where the Service Insight is exposed. + +Format: `[1024 - 65535]` (default `9114`) + + + +--- + +### -service-insight-tls-secret `` + +A Secret with a TLS certificate and key for TLS termination of the Service Insight endpoint. + +- If the argument is not set, the Service Insight endpoint will not use a TLS connection. +- If the argument is set, but NGINX Ingress Controller is not able to fetch the Secret from Kubernetes API, NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -spire-agent-address `` + +Specifies the address of a running Spire agent. **For use with NGINX Service Mesh only**. + +- If the argument is set, but NGINX Ingress Controller is unable to connect to the Spire Agent, NGINX Ingress Controller will fail to start. + + + + +--- + +### -enable-internal-routes + +Enable support for internal routes with NGINX Service Mesh. **For use with NGINX Service Mesh only**. + +Requires [-spire-agent-address](#cmdoption-spire-agent-address). + +- If the argument is set, but `spire-agent-address` is not provided, NGINX Ingress Controller will fail to start. + + + +--- + +### -enable-latency-metrics + +Enable collection of latency metrics for upstreams. +Requires [-enable-prometheus-metrics](#cmdoption-enable-prometheus-metrics). + + + +--- + +### -enable-app-protect + +Enables support for App Protect. + +Requires [-nginx-plus](#cmdoption-nginx-plus). + +- If the argument is set, but `nginx-plus` is set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-log-level `` + +Sets log level for App Protect. Allowed values: fatal, error, warn, info, debug, trace. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect](#cmdoption-enable-app-protect). + +- If the argument is set, but `nginx-plus` and `enable-app-protect` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -enable-app-protect-dos + +Enables support for App Protect DoS. + +Requires [-nginx-plus](#cmdoption-nginx-plus). + +- If the argument is set, but `nginx-plus` is set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-dos-debug + +Enable debugging for App Protect DoS. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). + +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-dos-max-daemons + +Max number of ADMD instances. + +Default `1`. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). + +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-dos-max-workers + +Max number of nginx processes to support. + +Default `Number of CPU cores in the machine`. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). + +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-dos-memory + +RAM memory size to consume in MB + +Default `50% of free RAM in the container or 80MB, the smaller`. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). + +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -ready-status + +Enables the readiness endpoint `/nginx-ready`. The endpoint returns a success code when NGINX has loaded all the config after the startup. + +Default `true`. + + + +--- + +### -ready-status-port + +The HTTP port for the readiness endpoint. + +Format: `[1024 - 65535]` (default `8081`) + +--- + +### -disable-ipv6 + +Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. + +Default `false`. + + + +--- + +### -default-http-listener-port + +Sets the port for the HTTP `default_server` listener. + +Default `80`. + + + +--- + +### -default-https-listener-port + +Sets the port for the HTTPS `default_server` listener. + +Default `443`. + + + +--- + +### -ssl-dynamic-reload + +Used to activate or deactivate lazy loading for SSL Certificates. + +The default value is `true`. + + + +--- + +### -weight-changes-dynamic-reload + +Enables the ability to change the weight distribution of two-way split clients without reloading NGINX. + +Requires [-nginx-plus](#cmdoption-nginx-plus). + +Using this feature may require increasing `map_hash_bucket_size`, `map_hash_max_size`, `variable_hash_bucket_size`, and `variable_hash_max_size` in the ConfigMap based on the number of two-way splits. + +The default value is `false`. + +- If the argument is set, but `nginx-plus` is set to false, NGINX Ingress Controller will ignore the flag. + + + +--- + +### -enable-telemetry-reporting + +Enable gathering and reporting of software telemetry. + +The default value is `true`. + + + +--- + +### -agent + +Enable NGINX Agent which can used with `-enable-app-protect` to send events to Security Monitoring. + +The default value is `false`. + + + +--- + +### -agent-instance-group + +Specify the instance group name to use for the NGINX Ingress Controller deployment when using `-agent`. + + diff --git a/content/includes/nic/configuration/global-configuration/configmap-resource.md b/content/includes/nic/configuration/global-configuration/configmap-resource.md new file mode 100644 index 000000000..afa0bcf17 --- /dev/null +++ b/content/includes/nic/configuration/global-configuration/configmap-resource.md @@ -0,0 +1,250 @@ +--- +title: ConfigMap resources +weight: 300 +toc: true +type: how-to +product: NIC +docs: DOCS-586 +--- + +When using F5 NGINX Ingress Controller, you can customize or fine tune NGINX behavior using ConfigMap resources. Examples include setting the number of worker processes or customizing the access log format. + +## Using ConfigMap + +1. The [Installation with Manifests]({{< relref "installation/installing-nic/installation-with-manifests.md" >}}) documentation deploy an empty ConfigMap while the default installation manifests specify it in the command-line arguments of the Ingress Controller. However, if you customized the manifests, to use ConfigMap, make sure to specify the ConfigMap resource to use the [command-line arguments]({{< ref "/nic/configuration/global-configuration/command-line-arguments" >}}) of NGINX Ingress Controller. + +1. Create a ConfigMap file with the name *nginx-config.yaml* and set the values +that make sense for your setup: + + ```yaml + kind: ConfigMap + apiVersion: v1 + metadata: + name: nginx-config + namespace: nginx-ingress + data: + proxy-connect-timeout: "10s" + proxy-read-timeout: "10s" + client-max-body-size: "2m" + ``` + + See the section [Summary of ConfigMap Keys](#summary-of-configmap-keys) for the explanation of the available ConfigMap keys (such as `proxy-connect-timeout` in this example). + +1. Create a new (or update the existing) ConfigMap resource: + + ```shell + kubectl apply -f nginx-config.yaml + ``` + + The NGINX configuration will be updated. + +--- + +## ConfigMap and Ingress annotations + +ConfigMap applies globally, meaning that it affects every Ingress resource. In contrast, annotations always apply to their Ingress resource. Annotations can override some ConfigMap keys: an example is that the `nginx.org/proxy-connect-timeout` annotations overrides the `proxy-connect-timeout` ConfigMap key. + +For more information, view the [Advanced configuration with annotations]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-annotations" >}}) topic. + +--- + +## ConfigMap and VirtualServer/VirtualServerRoute resources + +The ConfigMap affects every VirtualServer and VirtualServerRoute resources. However, the fields of those resources allow overriding some ConfigMap keys. For example, the `connect-timeout` field of the `upstream` overrides the `proxy-connect-timeout` ConfigMap key. + +For more information, view the [VirtualServer and VirtualServerRoute resources]({{< ref "/nic/configuration/virtualserver-and-virtualserverroute-resources" >}}) topic. + +--- + +## ConfigMap keys + +### Ingress Controller (Unrelated to NGINX Configuration) + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*external-status-address* | Sets the address to be reported in the status of Ingress resources. Requires the *-report-status* command-line argument. Overrides the *-external-service* argument. | N/A | [Reporting resource status]({{< ref "/nic/configuration/global-configuration/reporting-resources-status" >}}) | +{{}} + +--- + +### General customization + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*proxy-connect-timeout* | Sets the value of the [proxy_connect_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) and [grpc_connect_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout) directive. | *60s* | | +|*proxy-read-timeout* | Sets the value of the [proxy_read_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) and [grpc_read_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout) directive. | *60s* | | +|*proxy-send-timeout* | Sets the value of the [proxy_send_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout) and [grpc_send_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_send_timeout) directive. | *60s* | | +|*client-max-body-size* | Sets the value of the [client_max_body_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) directive. | *1m* | | +|*proxy-buffering* | Enables or disables [buffering of responses](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) from the proxied server. | *True* | | +|*proxy-buffers* | Sets the value of the [proxy_buffers](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directive. | Depends on the platform. | | +|*proxy-buffer-size* | Sets the value of the [proxy_buffer_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) and [grpc_buffer_size](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_buffer_size) directives. | Depends on the platform. | | +|*proxy-max-temp-file-size* | Sets the value of the [proxy_max_temp_file_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size) directive. | *1024m* | | +|*set-real-ip-from* | Sets the value of the [set_real_ip_from](https://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from) directive. | N/A | | +|*real-ip-header* | Sets the value of the [real_ip_header](https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header) directive. | *X-Real-IP* | | +|*real-ip-recursive* | Enables or disables the [real_ip_recursive](https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_recursive) directive. | *False* | | +|*default-server-return* | Configures the [return](https://nginx.org/en/docs/http/ngx_http_rewrite_module.html#return) directive in the default server, which handles a client request if none of the hosts of Ingress or VirtualServer resources match. The default value configures NGINX to return a 404 error page. You can configure a fixed response or a redirect. For example, *default-server-return: 302 https://nginx.org* will redirect a client to *https://nginx.org*. | *404* | | +|*server-tokens* | Enables or disables the [server_tokens](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens) directive. Additionally, with the NGINX Plus, you can specify a custom string value, including the empty string value, which disables the emission of the “Server” field. | *True* | | +|*worker-processes* | Sets the value of the [worker_processes](https://nginx.org/en/docs/ngx_core_module.html#worker_processes) directive. | *auto* | | +|*worker-rlimit-nofile* | Sets the value of the [worker_rlimit_nofile](https://nginx.org/en/docs/ngx_core_module.html#worker_rlimit_nofile) directive. | N/A | | +|*worker-connections* | Sets the value of the [worker_connections](https://nginx.org/en/docs/ngx_core_module.html#worker_connections) directive. | *1024* | | +|*worker-cpu-affinity* | Sets the value of the [worker_cpu_affinity](https://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity) directive. | N/A | | +|*worker-shutdown-timeout* | Sets the value of the [worker_shutdown_timeout](https://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout) directive. | N/A | | +|*server-names-hash-bucket-size* | Sets the value of the [server_names_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size) directive. | *256* | | +|*server-names-hash-max-size* | Sets the value of the [server_names_hash_max_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size) directive. | *1024* | | +|*map-hash-bucket-size* | Sets the value of the [map_hash_bucket_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_bucket_size) directive.| *256* | | +|*map-hash-max-size* | Sets the value of the [map_hash_max_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_max_size) directive. | *2048* | | +|*resolver-addresses* | Sets the value of the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) addresses. Note: If you use a DNS name (for example, *kube-dns.kube-system.svc.cluster.local* ) as a resolver address, NGINX Plus will resolve it using the system resolver during the start and on every configuration reload. If the name cannot be resolved or the DNS server doesn't respond, NGINX Plus will fail to start or reload. To avoid this, we recommend using IP addresses as resolver addresses instead of DNS names. Supported in NGINX Plus only. | N/A | [Support for Type ExternalName Services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services). | +|*resolver-ipv6* | Enables IPv6 resolution in the resolver. Supported in NGINX Plus only. | *True* | [Support for Type ExternalName Services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services). | +|*resolver-valid* | Sets the time NGINX caches the resolved DNS records. Supported in NGINX Plus only. | TTL value of a DNS record | [Support for Type ExternalName Services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services). | +|*resolver-timeout* | Sets the [resolver_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout) for name resolution. Supported in NGINX Plus only. | *30s* | [Support for Type ExternalName Services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services). | +|*keepalive-timeout* | Sets the value of the [keepalive_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) directive. | *75s* | | +|*keepalive-requests* | Sets the value of the [keepalive_requests](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests) directive. | *1000* | | +|*variables-hash-bucket-size* | Sets the value of the [variables_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables_hash_bucket_size) directive. | *256* | | +|*variables-hash-max-size* | Sets the value of the [variables-hash-max-size](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables_hash_max_size) directive. | *1024* | | +{{}} + +--- + +### Logging + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*error-log-level* | Sets the global [error log level](https://nginx.org/en/docs/ngx_core_module.html#error_log) for NGINX. | *notice* | | +|*access-log* | Sets the directive [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log). A syslog destination is the only valid value. The value will be set to its default in-case user tries to configure it with location other than a syslog. +| ``/dev/stdout main`` | ``syslog:server=localhost:514`` | +|*access-log-off* | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log). | *False* | | +|*default-server-access-log-off* | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log) for the default server. If access log is disabled globally (*access-log-off: "True"*), then the default server access log is always disabled. | *False* | | +|*log-format* | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for HTTP and HTTPS traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by *\n*). In that case, the Ingress Controller will replace every *\n* character with a space character. All *'* characters must be escaped. | See the [template file](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/internal/configs/version1/nginx.tmpl) for the access log. | [Custom Log Format](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/shared-examples/custom-log-format). | +|*log-format-escaping* | Sets the characters escaping for the variables of the log format. Supported values: *json* (JSON escaping), *default* (the default escaping) *none* (disables escaping). | *default* | | +|*stream-log-format* | Sets the custom [log format](https://nginx.org/en/docs/stream/ngx_stream_log_module.html#log_format) for TCP, UDP, and TLS Passthrough traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by *\n*). In that case, the Ingress Controller will replace every *\n* character with a space character. All *'* characters must be escaped. | See the [template file](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/internal/configs/version1/nginx.tmpl). | | +|*stream-log-format-escaping* | Sets the characters escaping for the variables of the stream log format. Supported values: *json* (JSON escaping), *default* (the default escaping) *none* (disables escaping). | *default* | | +{{}} + +--- + +### Request URI/Header manipulation + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*proxy-hide-headers* | Sets the value of one or more [proxy_hide_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directives. Example: *"nginx.org/proxy-hide-headers": "header-a,header-b"* | N/A | | +|*proxy-pass-headers* | Sets the value of one or more [proxy_pass_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directives. Example: *"nginx.org/proxy-pass-headers": "header-a,header-b"* | N/A | | +{{}} + +--- + +### Auth and SSL/TLS + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*redirect-to-https* | Sets the 301 redirect rule based on the value of the *http_x_forwarded_proto* header on the server block to force incoming traffic to be over HTTPS. Useful when terminating SSL in a load balancer in front of the Ingress Controller — see [115](https://github.com/nginx/kubernetes-ingress/issues/115) | *False* | | +|*ssl-redirect* | Sets an unconditional 301 redirect rule for all incoming HTTP traffic to force incoming traffic over HTTPS. | *True* | | +|*hsts* | Enables [HTTP Strict Transport Security (HSTS)](https://www.nginx.com/blog/http-strict-transport-security-hsts-and-nginx/) : the HSTS header is added to the responses from backends. The *preload* directive is included in the header. | *False* | | +|*hsts-max-age* | Sets the value of the *max-age* directive of the HSTS header. | *2592000* (1 month) | | +|*hsts-include-subdomains* | Adds the *includeSubDomains* directive to the HSTS header. | *False* | | +|*hsts-behind-proxy* | Enables HSTS based on the value of the *http_x_forwarded_proto* request header. Should only be used when TLS termination is configured in a load balancer (proxy) in front of the Ingress Controller. Note: to control redirection from HTTP to HTTPS configure the *nginx.org/redirect-to-https* annotation. | *False* | | +|*ssl-protocols* | Sets the value of the [ssl_protocols](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_protocols) directive. | *TLSv1 TLSv1.1 TLSv1.2* | | +|*ssl-prefer-server-ciphers* | Enables or disables the [ssl_prefer_server_ciphers](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_prefer_server_ciphers) directive. | *False* | | +|*ssl-ciphers* | Sets the value of the [ssl_ciphers](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers) directive. | *HIGH:!aNULL:!MD5* | | +|*ssl-dhparam-file* | Sets the content of the dhparam file. The controller will create the file and set the value of the [ssl_dhparam](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam) directive with the path of the file. | N/A | | +{{}} + +--- + +### Listeners + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*http2* | Enables HTTP/2 in servers with SSL enabled. | *False* | | +|*proxy-protocol* | Enables PROXY Protocol for incoming connections. | *False* | [Proxy Protocol](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/shared-examples/proxy-protocol). | +{{}} + +--- + +### Backend services (Upstreams) + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*lb-method* | Sets the [load balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify *"round_robin"*. | *"random two least_conn"* | | +|*max-fails* | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the *server* directive. | *1* | | +|*upstream-zone-size* | Sets the size of the shared memory [zone](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#zone) for upstreams. For NGINX, the special value 0 disables the shared memory zones. For NGINX Plus, shared memory zones are required and cannot be disabled. The special value 0 will be ignored. | *256k* for NGINX, *512k* for NGINX Plus | | +|*fail-timeout* | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the *server* directive. | *10s* | | +|*keepalive* | Sets the value of the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. Note that *proxy_set_header Connection "";* is added to the generated configuration when the value > 0. | *0* | | +{{}} + +--- + +### Zone Sync + +Zone Sync enables the [ngx_stream_zone_sync_module](https://nginx.org/en/docs/stream/ngx_stream_zone_sync_module.html) in NGINX Ingress Controller when NGINX Plus is used. Multiple replicas are required to effectively utililise this functionality. More information is available in the [How NGINX Plus Performs Zone Synchronization](https://docs.nginx.com/nginx/admin-guide/high-availability/zone_sync_details/) topic. + +Zone synchronization with TLS for NGINX Ingress Controller is not yet available with ConfigMap. If you would like to enable Zone Sync with TLS, please remove `zone-sync` from ConfigMap and add Zone Sync parameters via [`stream-snippets`]({{< ref "/configuration/ingress-resources/advanced-configuration-with-snippets.md" >}}) similar to [this example](https://github.com/nginx/kubernetes-ingress/blob/v4.0.1/examples/custom-resources/oidc/nginx-config.yaml) and adding the [zone_sync_ssl directive](https://nginx.org/en/docs/stream/ngx_stream_zone_sync_module.html#zone_sync_ssl) along with any other TLS parameters to the `stream-snippets`. + +You will also need to manually add the headless service, such as in [this example](https://github.com/nginx/kubernetes-ingress/blob/v4.0.1/examples/custom-resources/oidc/nginx-ingress-headless.yaml). + +{{< caution >}} +If you previously installed OIDC or used the `zone_sync` directive with `stream-snippets` in [v4.0.1](https://github.com/nginx/kubernetes-ingress/tree/v4.0.1) or earlier, and you plan to enable the `zone-sync` ConfigMap key, the `zone_sync` directive should be removed from `stream-snippets`. + +If you encounter the error `error [emerg] 13#13: "zone_sync" directive is duplicate in /etc/nginx/nginx.conf:164` it is likely due to `zone_sync` being enabled in both `stream-snippets` and the ConfigMap. Once upgraded, remove the [old headless service](https://github.com/nginx/kubernetes-ingress/blob/v4.0.1/examples/custom-resources/oidc/nginx-ingress-headless.yaml) deployed for OIDC. +{{< /caution >}} + + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*zone-sync* | Enables zone synchronization between NGINX Ingress Controller Pods. This autogenerates a [zone_sync_server](https://nginx.org/en/docs/stream/ngx_stream_zone_sync_module.html#zone_sync_server) and a headless service using the `ReplicaSet` or `DaemonSet` name. Please note that this headless service will be automatically cleaned up when uninstalling via Helm or by removing the value from the ConfigMap. The headless service will need to be manually removed if the `controller.customConfigMap` value is set via Helm or the deployment is uninstalled via Manifests. Each Ingress Controller manages its own headless service. NGINX Plus Required. | *False* | | +|*zone-sync-port* | Specifies the optional port on which NGINX Ingress Controller listens for zone sync traffic. NGINX Plus & `zone-sync` Required. | *12345* | | +|*zone-sync-resolver-addresses* | Configures optional addresses used in the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) directive for zone-sync. This field takes a comma separated list of addresses. NGINX Plus & `zone-sync` Required | `kube-dns.kube-system.svc.cluster.local` | | +|*zone-sync-resolver-ipv6* | Configures whether the optional [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) directive for zone-sync will look up IPv6 addresses. NGINX Plus & `zone-sync` Required | `true` | | +|*zone-sync-resolver-valid* | Configures an [NGINX time](https://nginx.org/en/docs/syntax.html) that the optional [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) directive for zone-sync will override the TTL value of responses from nameservers with. NGINX Plus & `zone-sync` Required | `5s` | | +{{}} + +--- + +### Snippets and custom templates + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*main-snippets* | Sets a custom snippet in main context. | N/A | | +|*http-snippets* | Sets a custom snippet in http context. | N/A | | +|*location-snippets* | Sets a custom snippet in location context. | N/A | | +|*server-snippets* | Sets a custom snippet in server context. | N/A | | +|*stream-snippets* | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/tcp-udp). | +|*main-template* | Sets the main NGINX configuration template. | By default the template is read from the file in the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | +|*ingress-template* | Sets the NGINX configuration template for an Ingress resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | +|*virtualserver-template* | Sets the NGINX configuration template for an VirtualServer resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | +|*transportserver-template* | Sets the NGINX configuration template for a TransportServer resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | +{{}} + +--- + +### Modules + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*otel-exporter-endpoint* | OTLP/gRPC endpoint that will accept [OpenTelemetry](https://opentelemetry.io) data. Set `otel-trace-in-http` to *"true"* to enable OpenTelemetry at the global level. | N/A | *"https://otel-collector:4317"* | +|*otel-exporter-header-name* | The name of a custom HTTP header to add to telemetry export request. `otel-exporter-endpoint` and `otel-exporter-header-value` required. | N/A | *"X-custom-header"* | +|*otel-exporter-header-value* | The value of a custom HTTP header to add to telemetry export request. `otel-exporter-endpoint` and `otel-exporter-header-name` required. | N/A | *"custom-value"* | +|*otel-service-name* | Sets the `service.name` attribute of the OTel resource. `otel-exporter-endpoint` required. | N/A | *"nginx-ingress-controller:nginx"* | +| *otel-trace-in-http* | Enables [OpenTelemetry](https://opentelemetry.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Set this to *"false"* to enable OpenTelemetry for individual routes with snippets. `otel-exporter-endpoint` required. | *"false"* | *"true"* | +|*opentracing* | Removed in v5.0.0. Enables [OpenTracing](https://opentracing.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Note: requires the Ingress Controller image with OpenTracing module and a tracer. See the [docs]({{< relref "/installation/integrations/opentracing.md" >}}) for more information. | *False* | | +|*opentracing-tracer* | Removed in v5.0.0. Sets the path to the vendor tracer binary plugin. | N/A | | +|*opentracing-tracer-config* | Removed in v5.0.0. Sets the tracer configuration in JSON format. | N/A | | +|*app-protect-compressed-requests-action* | Sets the *app_protect_compressed_requests_action* [global directive](/nginx-app-protect/configuration/#global-directives). | *drop* | | +|*app-protect-cookie-seed* | Sets the *app_protect_cookie_seed* [global directive](/nginx-app-protect/configuration/#global-directives). | Random automatically generated string | | +|*app-protect-failure-mode-action* | Sets the *app_protect_failure_mode_action* [global directive](/nginx-app-protect/configuration/#global-directives). | *pass* | | +|*app-protect-cpu-thresholds* | Sets the *app_protect_cpu_thresholds* [global directive](/nginx-app-protect/configuration/#global-directives). | *high=100 low=100* | | +|*app-protect-physical-memory-util-thresholds* | Sets the *app_protect_physical_memory_util_thresholds* [global directive](/nginx-app-protect/configuration/#global-directives). | *high=100 low=100* | | +|`app-protect-reconnect-period-seconds` | Sets the `app_protect_reconnect_period_seconds` [global directive](/nginx-app-protect/configuration/#global-directives). | `5` | | +|*app-protect-dos-log-format* | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for Dos Access log traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by *\n*). In that case, the Ingress Controller will replace every *\n* character with a space character. All *'* characters must be escaped. | `, vs_name_al=$app_protect_dos_vs_name, ip=$remote_addr, tls_fp=$app_protect_dos_tls_fp, outcome=$app_protect_dos_outcome, reason=$app_protect_dos_outcome_reason, policy_name=$app_protect_dos_policy_name, dos_version=$app_protect_dos_version, ip_tls=$remote_addr:$app_protect_dos_tls_fp,` | | +|*app-protect-dos-log-format-escaping* | Sets the characters escaping for the variables of the stream log format. Supported values: *json* (JSON escaping), *default* (the default escaping) *none* (disables escaping). | *default* | | +|*app-protect-dos-arb-fqdn* | Sets the *app-protect-dos-arb-fqdn* [directive](/nginx-app-protect-dos/directives-and-policy/learn-about-directives-and-policy/#arbitrator-fqdn-directive-app_protect_dos_arb_fqdn). | *svc-appprotect-dos-arb* | | +{{}} diff --git a/content/includes/nic/configuration/global-configuration/custom-templates.md b/content/includes/nic/configuration/global-configuration/custom-templates.md new file mode 100644 index 000000000..589ce5951 --- /dev/null +++ b/content/includes/nic/configuration/global-configuration/custom-templates.md @@ -0,0 +1,11 @@ +--- +docs: DOCS-587 +doctypes: +- '' +title: Custom templates +toc: true +weight: 500 +--- + + +F5 NGINX Ingress Controller uses templates to generate NGINX configuration for Ingress resources, VirtualServer resources and the main NGINX configuration file. You can customize the templates and apply them via the ConfigMap. See the [corresponding example](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/shared-examples/custom-templates). diff --git a/content/includes/nic/configuration/global-configuration/globalconfiguration-resource.md b/content/includes/nic/configuration/global-configuration/globalconfiguration-resource.md new file mode 100644 index 000000000..605ab301f --- /dev/null +++ b/content/includes/nic/configuration/global-configuration/globalconfiguration-resource.md @@ -0,0 +1,183 @@ +--- +docs: DOCS-588 +doctypes: +- '' +title: GlobalConfiguration resource +toc: true +weight: 200 +--- + +This page explains how to use the GlobalConfiguration resource to define the global configuration parameters of F5 NGINX Ingress Controller. + +The resource supports configuring listeners for TCP and UDP load balancing, and is implemented as a [Custom resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + +Listeners are required by [TransportServer resources]({{< relref "/configuration/transportserver-resource.md" >}}) and can be used to [configure custom listeners for VirtualServers]({{< relref "tutorials/virtual-server-with-custom-listener-ports.md" >}}). + +--- + +## Prerequisites + +When [installing NGINX Ingress Controller using Manifests]({{< relref "/installation/installing-nic/installation-with-manifests.md" >}}), you need to reference a GlobalConfiguration resource in the [`-global-configuration`](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments#cmdoption-global-configuration) command-line argument. NGINX Ingress Controller only needs one GlobalConfiguration resource. + +--- + +## GlobalConfiguration specification + +The GlobalConfiguration resource defines the global configuration parameters of the Ingress Controller. Below is an example: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: GlobalConfiguration +metadata: + name: nginx-configuration + namespace: nginx-ingress +spec: + listeners: + - name: dns-udp + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP + - name: http-8083 + port: 8083 + protocol: HTTP + - name: https-8443 + port: 8443 + protocol: HTTP + ssl: true +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +| *listeners* | A list of listeners. | [listener](#listener) | No | +{{}} + +### Listener + +The `listeners:` key defines a listener (a combination of a protocol and a port) that NGINX will use to accept traffic for a [TransportServer]({{< relref "/configuration/transportserver-resource.md" >}}) and a [VirtualServer]({{< relref "/configuration/virtualserver-and-virtualserverroute-resources.md" >}}): + +```yaml +- name: dns-tcp + port: 5353 + protocol: TCP +- name: http-8083 + port: 8083 + protocol: HTTP +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +| *name* | The name of the listener. Must be a valid DNS label as defined in RFC 1035. For example, ``hello`` and ``listener-123`` are valid. The name must be unique among all listeners. The name ``tls-passthrough`` is reserved for the built-in TLS Passthrough listener and cannot be used. | *string* | Yes | +| *port* | The port of the listener. The port must fall into the range ``1..65535`` with the following exceptions: ``80``, ``443``, the [status port](/nginx-ingress-controller/logging-and-monitoring/status-page), the [Prometheus metrics port](/nginx-ingress-controller/logging-and-monitoring/prometheus). Among all listeners, only a single combination of a port-protocol is allowed. | *int* | Yes | +| *protocol* | The protocol of the listener. Supported values: ``TCP``, ``UDP`` and ``HTTP``. | *string* | Yes | +| *ssl* | Configures the listener with SSL. This is currently only supported for ``HTTP`` listeners. Default value is ``false`` | *bool* | No | +| *ipv4* | Specifies the IPv4 address to listen on. | *string* | No | +| *ipv6* | Specifies the IPv6 address to listen on. | *string* | No | + +{{}} + +--- + +## Using GlobalConfiguration + +You can use the usual `kubectl` commands to work with a GlobalConfiguration resource. + +For example, the following command creates a GlobalConfiguration resource defined in `global-configuration.yaml` with the name `nginx-configuration`: + +```shell +kubectl apply -f global-configuration.yaml +``` +```shell +globalconfiguration.k8s.nginx.org/nginx-configuration created +``` + +Assuming the namespace of the resource is `nginx-ingress`, you can get the resource by running: + +```shell +kubectl get globalconfiguration nginx-configuration -n nginx-ingress +``` +```shell +NAME AGE +nginx-configuration 13s +``` + +With `kubectl get` and similar commands, you can use the short name `gc` instead of `globalconfiguration`. + +--- + +### Validation + +Two types of validation are available for the GlobalConfiguration resource: + +- *Structural validation* by `kubectl` and Kubernetes API server. +- *Comprehensive validation* by NGINX Ingress Controller. + + +#### Structural validation + +The custom resource definition for the GlobalConfiguration includes structural OpenAPI schema which describes the type of every field of the resource. + +If you try to create (or update) a resource that violates the structural schema (for example, you use a string value for the port field of a listener), `kubectl` and Kubernetes API server will reject such a resource: + +- Example of `kubectl` validation: + + ```shell + kubectl apply -f global-configuration.yaml + ``` + ```text + error: error validating "global-configuration.yaml": error validating data: ValidationError(GlobalConfiguration.spec.listeners[0].port): invalid type for org.nginx.k8s.v1.GlobalConfiguration.spec.listeners.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + ``` + +- Example of Kubernetes API server validation: + + ```shell + kubectl apply -f global-configuration.yaml --validate=false + ``` + ```text + The GlobalConfiguration "nginx-configuration" is invalid: []: Invalid value: map[string]interface {}{ ... }: validation failure list: + spec.listeners.port in body must be of type integer: "string" + ``` + +If a resource is not rejected (it doesn't violate the structural schema), NGINX Ingress Controller will validate it further. + +#### Comprehensive validation + +NGINX Ingress Controller validates the fields of a GlobalConfiguration resource. If a GlobalConfiguration resource is partially invalid, NGINX Ingress Controller use the valid listeners and emit events about invalid listeners. + +You can check if the Ingress Controller successfully applied the configuration for a GlobalConfiguration. For our `nginx-configuration` GlobalConfiguration, we can run: + +```shell +kubectl describe gc nginx-configuration -n nginx-ingress +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Updated 11s nginx-ingress-controller GlobalConfiguration nginx-ingress/nginx-configuration was updated +``` + +The events section includes a Normal event with the Updated reason that informs us that the configuration was successfully applied. + +If you create a GlobalConfiguration `nginx-configuration` with two or more listeners that have the same protocol UDP and port 53, you will get: + +```shell +kubectl describe gc nginx-configuration -n nginx-ingress +``` +```text +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Updated 55s nginx-ingress-controller GlobalConfiguration nginx-ingress/nginx-configuration was updated + Warning AddedOrUpdatedWithError 6s nginx-ingress-controller GlobalConfiguration nginx-ingress/nginx-configuration is invalid and was rejected: spec.listeners: Duplicate value: "Duplicated port/protocol combination 53/UDP" +``` + +The events section includes a Warning event with the AddedOrUpdatedWithError reason. + + +## Using IPV4 and IPV6 Addresses with GlobalConfiguration + +You can customize the IPv4 and IPv6 Address listeners in the global configuration and apply them to your VirtualServer resources. See the corresponding example [here](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/custom-ip-listeners/virtualserver/) diff --git a/content/includes/nic/configuration/global-configuration/mgmt-configmap-resource.md b/content/includes/nic/configuration/global-configuration/mgmt-configmap-resource.md new file mode 100644 index 000000000..e1d9a120c --- /dev/null +++ b/content/includes/nic/configuration/global-configuration/mgmt-configmap-resource.md @@ -0,0 +1,50 @@ +--- +docs: DOCS-586 +doctypes: +- '' +title: Management ConfigMap resource +toc: true +weight: 300 +--- + +When using F5 NGINX Ingress Controller with NGINX Plus, it is required to pass a [command line argument]({{< ref "/nic/configuration/global-configuration/command-line-arguments" >}}) to NGINX Ingress Controller, `--mgmt-configmap=` which specifies the ConfigMap to use. The minimal required ConfigMap must have a `license-token-secret-name` key. Helm users will not need to create this map or pass the argument, it will be created with a Helm install. + +--- + +1. Create a ConfigMap file with the name *nginx-config-mgmt.yaml* and set the values +that make sense for your setup: + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: nginx-config-mgmt + namespace: nginx-ingress + data: + license-token-secret-name: "license-token" + ``` +1. Create a new (or update the existing) ConfigMap resource: + + ```shell + kubectl apply -f nginx-config-mgmt.yaml + ``` + + The [NGINX Management](https://nginx.org/en/docs/ngx_mgmt_module.html) block configuration will be updated. +--- +## Management ConfigMap keys + +{{}} +|ConfigMap Key | Description | Default | +| ---| ---| ---| +|*license-token-secret-name* | Configures the secret used in the [license_token](https://nginx.org/en/docs/ngx_mgmt_module.html#license_token) directive. This key assumes the secret is in the Namespace that NGINX Ingress Controller is deployed in. The secret must be of type `nginx.com/license` with the base64 encoded JWT in the `license.jwt` key. | N/A | +|*ssl-verify* | Configures the [ssl_verify](https://nginx.org/en/docs/ngx_mgmt_module.html#ssl_verify) directive, which enables or disables verification of the usage reporting endpoint certificate. | `true` | +|*enforce-initial-report* | Configures the [enforce_initial_report](https://nginx.org/en/docs/ngx_mgmt_module.html#enforce_initial_report) directive, which enables or disables the 180-day grace period for sending the initial usage report. | `false` | +|*usage-report-endpoint* | Configures the endpoint of the [usage_report](https://nginx.org/en/docs/ngx_mgmt_module.html#usage_report) directive. This is used to configure the endpoint NGINX uses to send usage reports to NIM. | `product.connect.nginx.com` | +|*usage-report-interval* | Configures the interval of the [usage_report](https://nginx.org/en/docs/ngx_mgmt_module.html#usage_report) directive. This specifies the frequency that usage reports are sent. This field takes an [NGINX time](https://nginx.org/en/docs/syntax.html). | `1h` | +|*usage-report-proxy-host* | Configures the host name of the [proxy](https://nginx.org/en/docs/ngx_mgmt_module.html#proxy) directive with optional port. | N/A | +|*ssl-trusted-certificate-secret-name* | Configures the secret used to create the file(s) referenced the in [ssl_trusted_certifcate](https://nginx.org/en/docs/ngx_mgmt_module.html#ssl_trusted_certificate), and [ssl_crl](https://nginx.org/en/docs/ngx_mgmt_module.html#ssl_crl) directives. This key assumes the secret is in the Namespace that NGINX Ingress Controller is deployed in. The secret must be of type `nginx.org/ca`, where the `ca.crt` key contains a base64 encoded trusted cert, and the optional `ca.crl` key can contain a base64 encoded CRL. If the optional `ca.crl` key is supplied, it will configure the NGINX `ssl_crl` directive. | N/A | +|*ssl-certificate-secret-name* | Configures the secret used to create the `ssl_certificate` and `ssl_certificate_key` directives. This key assumes the secret is in the Namespace that NGINX Ingress Controller is deployed in. The secret must be of type `kubernetes.io/tls`| N/A | +|*resolver-addresses* | Configures addresses used in the mgmt block [resolver](https://nginx.org/en/docs/ngx_mgmt_module.html#resolver) directive. This field takes a comma separated list of addresses. | N/A | +|*resolver-ipv6* | Configures whether the mgmt block [resolver](https://nginx.org/en/docs/ngx_mgmt_module.html#resolver) directive will look up IPv6 addresses. | `true` | +|*resolver-valid* | Configures an [NGINX time](https://nginx.org/en/docs/syntax.html) that the mgmt block [resolver](https://nginx.org/en/docs/ngx_mgmt_module.html#resolver) directive will override the TTL value of responses from nameservers with. | N/A | +{{}} diff --git a/content/includes/nic/configuration/global-configuration/reporting-resources-status.md b/content/includes/nic/configuration/global-configuration/reporting-resources-status.md new file mode 100644 index 000000000..8e36733eb --- /dev/null +++ b/content/includes/nic/configuration/global-configuration/reporting-resources-status.md @@ -0,0 +1,195 @@ +--- +docs: DOCS-589 +doctypes: +- '' +title: Reporting resource status +toc: true +weight: 600 +--- + +This page describes how to view the status of resources managed by F5 NGINX Ingress Controller. + +## Ingress resources + +An Ingress resource status includes the address (an IP address or a DNS name), through which the hosts of that Ingress resource are publicly accessible. + +You can see the address in the output of the `kubectl get ingress` command, in the ADDRESS column, as shown below: + +```shell +kubectl get ingresses +``` +```text +NAME HOSTS ADDRESS PORTS AGE +cafe-ingress cafe.example.com 12.13.23.123 80, 443 2m +``` + +NGINX Ingress Controller must be configured to report an Ingress status: + +1. Use the command-line flag `-report-ingress-status`. +1. Define a source for an external address. This can be either of: + 1. A user defined address, specified in the `external-status-address` ConfigMap key. + 1. A Service of the type LoadBalancer configured with an external IP or address and specified by the `-external-service` command-line flag. + +View the [ConfigMap keys](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) and [Command-line arguments](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments) topics for more information. + +{{< note >}} NGINX Ingress Controller does not clear the status of Ingress resources when it is being shut down. {{< /note >}} + +## VirtualServer and VirtualServerRoute resources + +A VirtualServer or VirtualServerRoute resource includes the status field with information about the state of the resource and the IP address, through which the hosts of that resource are publicly accessible. + +You can see the status in the output of the `kubectl get virtualservers` or `kubectl get virtualserverroutes` commands as shown below: + +```shell +kubectl get virtualservers +``` +```text + NAME STATE HOST IP PORTS AGE + cafe Valid cafe.example.com 12.13.23.123 [80,443] 34s +``` + +To see an external hostname address associated with a VirtualServer resource, use the `-o wide` option: + +```shell +kubectl get virtualservers -o wide +``` +```text + NAME STATE HOST IP EXTERNALHOSTNAME PORTS AGE + cafe Valid cafe.example.com ae430f41a1a0042908655abcdefghijkl-12345678.eu-west-2.elb.amazonaws.com [80,443] 106s +``` + +{{< note >}} If there are multiple addresses, only the first one is shown. {{< /note >}} + +In order to see additional addresses or extra information about the `Status` of the resource, use the following command: + +```shell +kubectl describe virtualserver +``` +```text +... +Status: + External Endpoints: + Ip: 12.13.23.123 + Ports: [80,443] + Message: Configuration for cafe/cafe was added or updated + Reason: AddedOrUpdated + State: Valid +``` + +### Status specification + +The following fields are reported in both VirtualServer and VirtualServerRoute status: + +{{}} +|Field | Description | Type | +| ---| ---| --- | +|*State* | Current state of the resource. Can be ``Valid``, ``Warning`` an ``Invalid``. For more information, refer to the ``message`` field. | *string* | +|*Reason* | The reason of the last update. | *string* | +|*Message* | Additional information about the state. | *string* | +|*ExternalEndpoints* | A list of external endpoints for which the hosts of the resource are publicly accessible. | *[externalEndpoint](#externalendpoint)* | +{{}} + +The *ReferencedBy* field is reported for the VirtualServerRoute status only: + +{{}} +|Field | Description | Type | +| ---| ---| --- | +| *ReferencedBy* | The VirtualServer that references this VirtualServerRoute. Format as ``namespace/name`` | *string* | +{{}} + +### externalEndpoint + +{{}} +|Field | Description | Type | +| ---| ---| --- | +|``IP`` | The external IP address. | ``string`` | +|``Hostname`` | The external LoadBalancer Hostname address. | ``string`` | +|``Ports`` | A list of external ports. | ``string`` | +{{}} + +NGINX Ingress Controller must be configured to report a VirtualServer or VirtualServerRoute status: + +1. If you want NGINX Ingress Controller to report the `externalEndpoints`, define a source for an external address (The rest of the fields will be reported without the external address configured). This can be: + 1. A user defined address, specified in the `external-status-address` ConfigMap key. + 1. A Service of the type LoadBalancer configured with an external IP or address and specified by the `-external-service` command-line flag. + +View the [ConfigMap keys](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) and [Command-line arguments](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments) topics for more information. + +{{< note >}} NGINX Ingress Controller does not clear the status of VirtualServer and VirtualServerRoute resources when it is being shut down. {{< /note >}} + +## Policy resources + +A Policy resource includes the status field with information about the state of the resource. + +You can see the status in the output of the `kubectl get policy` command as shown below: + +```shell +kubectl get policy +``` +```text + NAME STATE AGE + webapp-policy Valid 30s +``` + +In order to see additional addresses or extra information about the `Status` of the resource, use the following command: + +```shell +kubectl describe policy +``` +```text +... +Status: + Message: Configuration for default/webapp-policy was added or updated + Reason: AddedOrUpdated + State: Valid +``` + +### Status specification + +The following fields are reported in Policy status: + +{{}} +|Field | Description | Type | +| ---| ---| --- | +|``State`` | Current state of the resource. Can be ``Valid`` or ``Invalid``. For more information, refer to the ``message`` field. | ``string`` | +|``Reason`` | The reason of the last update. | ``string`` | +|``Message`` | Additional information about the state. | ``string`` | +{{}} + +## TransportServer resources + +A TransportServer resource includes the status field with information about the state of the resource. + +You can see the status in the output of the `kubectl get transportserver` command as shown below: + +```shell +kubectl get transportserver +``` +```text + NAME STATE REASON AGE + dns-tcp Valid AddedOrUpdated 47m +``` + +To see additional addresses or extra information about the `Status` of the resource, use the following command: + +```shell +kubectl describe transportserver +``` +```text +Status: + Message: Configuration for default/dns-tcp was added or updated + Reason: AddedOrUpdated + State: Valid +``` + +### Status specification + +The following fields are reported in TransportServer status: + +{{}} +|Field | Description | Type | +| ---| ---| --- | +| *State* | Current state of the resource. Can be ``Valid``, ``Warning`` or ``Invalid``. For more information, refer to the ``message`` field. | *string* | +| *Reason* | The reason of the last update. | *string* | +| *Message* | Additional information about the state. | *string* | +{{}} diff --git a/content/includes/nic/configuration/host-and-listener-collisions.md b/content/includes/nic/configuration/host-and-listener-collisions.md new file mode 100644 index 000000000..502ef830c --- /dev/null +++ b/content/includes/nic/configuration/host-and-listener-collisions.md @@ -0,0 +1,168 @@ +--- +title: Host and Listener collisions +toc: true +weight: 800 +docs: DOCS-590 +--- + +This document explains how F5 NGINX Ingress Controller handles host and listener collisions between resources. + +--- + +## Winner Selection Algorithm + +If multiple resources contend for the same host or listener, NGINX Ingress Controller will pick the winner based on the `creationTimestamp` of the resources: the oldest resource will win. In case there are more than one oldest resource (their `creationTimestamp` is the same), NGINX Ingress Controller will choose the resource with the lexicographically smallest `uid`. + +{{< note >}} The `creationTimestamp` and `uid` fields are part of the [ObjectMeta](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/object-meta/) resource. {{< /note >}} + +--- + +## Host collisions + +A host collision occurs when multiple Ingress, VirtualServer, and TransportServer (configured for TLS Passthrough) resources configure the same `host`. NGINX Ingress Controller has two strategies for handling host collisions: + +- Choosing a single "winner" resource to handle the host. +- Merging the configuration of the conflicting resources. + +--- + +### Choosing the winner + +Consider the following two resources: + +- `cafe-ingress` Ingress: + + ```yaml + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cafe-ingress + spec: + ingressClassName: nginx + rules: + - host: cafe.example.com + . . . + ``` + +- `cafe-virtual-server` VirtualServer: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: VirtualServer + metadata: + name: cafe-virtual-server + spec: + host: cafe.example.com + . . . + ``` + +If a user creates both resources in the cluster, a host collision will occur. NGINX Ingress Controller will pick the winner using the [winner selection algorithm](#winner-selection-algorithm). + +If `cafe-virtual-server` was created first, it will win the host `cafe.example.com` and NGINX Ingress Controller will reject `cafe-ingress`. This will be reflected in the events and in the resource's status field: + +```shell +kubectl describe vs cafe-virtual-server +``` +```text +... +Status: + ... + Message: Configuration for default/cafe-virtual-server was added or updated + Reason: AddedOrUpdated + State: Valid +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 9s nginx-ingress-controller Configuration for default/cafe-virtual-server was added or updated +``` + +```shell +kubectl describe ingress cafe-ingress +``` +```text +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 66s nginx-ingress-controller All hosts are taken by other resources +``` + +Similarly, if `cafe-ingress` was created first, it will win `cafe.example.com` and NGINX Ingress Controller will reject `cafe-virtual-server`. + +{{< note >}} You can configure multiple hosts for Ingress resources, and its possible that an Ingress resource can be the winner for some of its hosts and a loser for the others. + +For example, if `cafe-ingress` had an additional rule host rule for `pub.example.com`, NGINX Ingress Controller would not reject the Ingress. Instead, it would allow `cafe-ingress` to handle `pub.example.com`. {{< /note >}} + +--- + +### Merging configuration for the same host + +It is possible to merge configuration for multiple Ingress resources for the same host. One common use case for this approach is distributing resources across multiple namespaces. + +The [Cross-namespace configuration]({{< ref "/nic/configuration/ingress-resources/cross-namespace-configuration.md">}}) topic has more information. + +It is *not* possible to merge the configurations for multiple VirtualServer resources for the same host. However, you can split the VirtualServers into multiple VirtualServerRoute resources, which a single VirtualServer can then reference. See the [corresponding example](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/cross-namespace-configuration) on GitHub. + +It is *not* possible to merge configuration for multiple TransportServer resources. + +--- + +## Listener/Host collisions + +Listener/Host collisions occur when multiple TransportServer resources (configured for TCP/UDP load balancing) specify the same combination of `spec.listener.name` and `spec.host`. + +The combination of `spec.listener.name` and `spec.host` must be unique among all TransportServer resources. If two TransportServer resources specify the same spec.listener.name and spec.host, one of them will be rejected to prevent conflicts. In the case where spec.host is not specified, it is considered an empty string. + +NGINX Ingress Controller will choose the winner, which will own that listener and host combination. + +--- + +### Choosing the winner + +Consider the following two resources: + +- `tcp-1` TransportServer: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: tcp-1 + spec: + host: dns.example.com + listener: + name: dns-tcp + protocol: TCP + . . . + ``` + +- `tcp-2` TransportServer: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: tcp-2 + spec: + host: dns.example.com + listener: + name: dns-tcp + protocol: TCP + . . . + ``` + +If a user creates both resources in the cluster, a listener collision will occur. As a result, NGINX Ingress Controller will pick the winner using the [winner selection algorithm](#winner-selection-algorithm). + +In our example, if `tcp-1` was created first, it will win the listener `dns-tcp` and NGINX Ingress Controller will reject `tcp-2`. This will be reflected in the events and in the resource's status field: + +```shell +kubectl describe ts tcp-2 +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 10s nginx-ingress-controller Listener dns-tcp is taken by another resource +``` + +Similarly, if `tcp-2` was created first, it will win `dns-tcp` and NGINX Ingress Controller will reject `tcp-1`. diff --git a/content/includes/nic/configuration/ingress-resources/_index.md b/content/includes/nic/configuration/ingress-resources/_index.md new file mode 100644 index 000000000..8a2b5b282 --- /dev/null +++ b/content/includes/nic/configuration/ingress-resources/_index.md @@ -0,0 +1,8 @@ +--- +title: Ingress resources +description: +weight: 200 +menu: + docs: + parent: NGINX Ingress Controller +--- diff --git a/content/includes/nic/configuration/ingress-resources/advanced-configuration-with-annotations.md b/content/includes/nic/configuration/ingress-resources/advanced-configuration-with-annotations.md new file mode 100644 index 000000000..ecc12435f --- /dev/null +++ b/content/includes/nic/configuration/ingress-resources/advanced-configuration-with-annotations.md @@ -0,0 +1,225 @@ +--- +docs: DOCS-591 +doctypes: +- '' +title: Advanced configuration with Annotations +toc: true +weight: 200 +--- + +This topic explains how to enable advanced features in F5 NGINX Ingress Controller with Annotations. + +The Ingress resource can use basic NGINX features such as host or path-based routing and TLS termination. Advanced features like rewriting the request URI or inserting additional response headers can be enabled with Annotations. + +Outside of advanced features, Annotations are necessary for customizing NGINX behavior such as setting the value of connection timeouts. + +Customization is also available through the [ConfigMap]({{< relref "/configuration/global-configuration/configmap-resource.md" >}}) resources: Annotations take priority. + +## Using Annotations + +This example uses Annotations to customize the configuration for an Ingress resource: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: cafe-ingress-with-annotations + annotations: + nginx.org/proxy-connect-timeout: "30s" + nginx.org/proxy-read-timeout: "20s" + nginx.org/client-max-body-size: "4m" + nginx.org/server-snippets: | + location / { + return 302 /coffee; + } +spec: + rules: + - host: cafe.example.com + http: + paths: + - path: /tea + pathType: Prefix + backend: + service: + name: tea-svc + port: + number: 80 + - path: /coffee + pathType: Prefix + backend: + service: + name: coffee-svc + port: + number: 80 +``` + +## Validation + +NGINX Ingress Controller validates the annotations of Ingress resources. If an Ingress is invalid, NGINX Ingress Controller will reject it: the Ingress will continue to exist in the cluster, but NGINX Ingress Controller will ignore it. + +You can check if NGINX Ingress Controller successfully applied the configuration for an Ingress resource. For the example `cafe-ingress-with-annotations` Ingress, you can run: + +```shell +kubectl describe ing cafe-ingress-with-annotations +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 3s nginx-ingress-controller Configuration for default/cafe-ingress-with-annotations was added or updated +``` + +The events section includes a Normal event with the AddedOrUpdated reason that informs us that the configuration was successfully applied. + +If you create an invalid Ingress, NGINX Ingress Controller will reject it and emit a Rejected event. For example, if you create an Ingress `cafe-ingress-with-annotations`, with an annotation `nginx.org/redirect-to-https` set to `yes please` instead of `true`, you will get: + +```shell +kubectl describe ing cafe-ingress-with-annotations +``` +```text +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 13s nginx-ingress-controller annotations.nginx.org/redirect-to-https: Invalid value: "yes please": must be a boolean +``` + +Note how the events section includes a Warning event with the Rejected reason. + +{{< note >}} If you make an existing Ingress invalid, NGINX Ingress Controller will reject it and remove the corresponding configuration from NGINX. {{< /note >}} + +The `nginx.com/jwt-token` Ingress annotation has limited validation. + +## Summary of Annotations + +The table below summarizes the available annotations. + +{{< note >}} Annotations that start with `nginx.com` are only supported with NGINX Plus. {{< /note >}} + +### General customization + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/proxy-connect-timeout* | *proxy-connect-timeout* | Sets the value of the [proxy_connect_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) and [grpc_connect_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout) directive. | *60s* | | +| *nginx.org/proxy-read-timeout* | *proxy-read-timeout* | Sets the value of the [proxy_read_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) and [grpc_read_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout) directive. | *60s* | | +| *nginx.org/proxy-send-timeout* | *proxy-send-timeout* | Sets the value of the [proxy_send_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout) and [grpc_send_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_send_timeout) directive. | *60s* | | +| *nginx.org/client-max-body-size* | *client-max-body-size* | Sets the value of the [client_max_body_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) directive. | *1m* | | +| *nginx.org/proxy-buffering* | *proxy-buffering* | Enables or disables [buffering of responses](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) from the proxied server. | *True* | | +| *nginx.org/proxy-buffers* | *proxy-buffers* | Sets the value of the [proxy_buffers](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directive. | Depends on the platform. | | +| *nginx.org/proxy-buffer-size* | *proxy-buffer-size* | Sets the value of the [proxy_buffer_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) and [grpc_buffer_size](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_buffer_size) directives. | Depends on the platform. | | +| *nginx.org/proxy-max-temp-file-size* | *proxy-max-temp-file-size* | Sets the value of the [proxy_max_temp_file_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size) directive. | *1024m* | | +| *nginx.org/server-tokens* | *server-tokens* | Enables or disables the [server_tokens](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens) directive. Additionally, with the NGINX Plus, you can specify a custom string value, including the empty string value, which disables the emission of the “Server” field. | *True* | | +| *nginx.org/path-regex* | N/A | Enables regular expression modifiers for Ingress path parameter. This translates to the NGINX [location](https://nginx.org/en/docs/http/ngx_http_core_module.html#location) directive. You can specify one of these values: "case_sensitive", "case_insensitive", or "exact". The annotation is applied to the entire Ingress resource and its paths. While using Master and Minion Ingresses i.e. Mergeable Ingresses, this annotation can be specified on Minion types. The `path-regex` annotation specified on Master is ignored, and has no effect on paths defined on Minions. | N/A | [path-regex](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/path-regex) | +{{}} + +### Request URI/Header Manipulation + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/proxy-hide-headers* | *proxy-hide-headers* | Sets the value of one or more [proxy_hide_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directives. Example: ``"nginx.org/proxy-hide-headers": "header-a,header-b"* | N/A | | +| *nginx.org/proxy-pass-headers* | *proxy-pass-headers* | Sets the value of one or more [proxy_pass_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directives. Example: ``"nginx.org/proxy-pass-headers": "header-a,header-b"* | N/A | | +| *nginx.org/rewrites* | N/A | Configures URI rewriting using [proxy_pass](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass) directive. | N/A | [rewrites](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/rewrites) | +|*nginx.org/proxy-set-headers* | N/A | Enables customization of proxy headers and values using the [proxy_set_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) directive. Example: ``"nginx.org/proxy-set-headers": "header-a: valueA,header-b: valueB,header-c: valueC"`` | N/A | [Proxy Set Headers](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/proxy-set-headers). | +{{}} + +### Auth and SSL/TLS + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/redirect-to-https* | *redirect-to-https* | Sets the 301 redirect rule based on the value of the ``http_x_forwarded_proto* header on the server block to force incoming traffic to be over HTTPS. Useful when terminating SSL in a load balancer in front of NGINX Ingress Controller — see [115](https://github.com/nginx/kubernetes-ingress/issues/115) | *False* | | +| *ingress.kubernetes.io/ssl-redirect* | *ssl-redirect* | Sets an unconditional 301 redirect rule for all incoming HTTP traffic to force incoming traffic over HTTPS. | *True* | | +| *nginx.org/hsts* | *hsts* | Enables [HTTP Strict Transport Security (HSTS)](https://www.nginx.com/blog/http-strict-transport-security-hsts-and-nginx/)\ : the HSTS header is added to the responses from backends. The ``preload* directive is included in the header. | *False* | | +| *nginx.org/hsts-max-age* | *hsts-max-age* | Sets the value of the ``max-age* directive of the HSTS header. | *2592000* (1 month) | | +| *nginx.org/hsts-include-subdomains* | *hsts-include-subdomains* | Adds the ``includeSubDomains* directive to the HSTS header. | *False* | | +| *nginx.org/hsts-behind-proxy* | *hsts-behind-proxy* | Enables HSTS based on the value of the ``http_x_forwarded_proto* request header. Should only be used when TLS termination is configured in a load balancer (proxy) in front of NGINX Ingress Controller. Note: to control redirection from HTTP to HTTPS configure the ``nginx.org/redirect-to-https* annotation. | *False* | | +| *nginx.org/basic-auth-secret* | N/A | Specifies a Secret resource with a user list for HTTP Basic authentication. | N/A | | +| *nginx.org/basic-auth-realm* | N/A | Specifies a realm. | N/A | | +| *nginx.com/jwt-key* | N/A | Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/jwt). | +| *nginx.com/jwt-realm* | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/jwt). | +| *nginx.com/jwt-token* | N/A | Specifies a variable that contains a JSON Web Token. | By default, a JWT is expected in the ``Authorization* header as a Bearer Token. | [Support for JSON Web Tokens (JWTs)](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/jwt). | +| *nginx.com/jwt-login-url* | N/A | Specifies a URL to which a client is redirected in case of an invalid or missing JWT. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/jwt). | +{{}} + +### Listeners + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/listen-ports* | N/A | Configures HTTP ports that NGINX will listen on. | *[80]* | | +| *nginx.org/listen-ports-ssl* | N/A | Configures HTTPS ports that NGINX will listen on. | *[443]* | | +{{}} + +### Backend services (Upstreams) + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/lb-method* | *lb-method* | Sets the [load balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify ``"round_robin"``. | *"random two least_conn"* | | +| *nginx.org/ssl-services* | N/A | Enables HTTPS or gRPC over SSL when connecting to the endpoints of services. | N/A | [ssl-services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/ssl-services) | +| *nginx.org/grpc-services* | N/A | Enables gRPC for services. Note: requires HTTP/2 (see ``http2* ConfigMap key); only works for Ingresses with TLS termination enabled. | N/A | [grpc-services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/grpc-services) | +| *nginx.org/websocket-services* | N/A | Enables WebSocket for services. | N/A | [websocket](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/websocket) | +| *nginx.org/max-fails* | *max-fails* | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the ``server* directive. | *1* | | +| *nginx.org/max-conns* | N\A | Sets the value of the [max_conns](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_conns) parameter of the ``server* directive. | *0* | | +| *nginx.org/upstream-zone-size* | *upstream-zone-size* | Sets the size of the shared memory [zone](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#zone) for upstreams. For NGINX, the special value 0 disables the shared memory zones. For NGINX Plus, shared memory zones are required and cannot be disabled. The special value 0 will be ignored. | *256K* | | +| *nginx.org/fail-timeout* | *fail-timeout* | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the ``server* directive. | *10s* | | +| *nginx.com/sticky-cookie-services* | N/A | Configures session persistence. | N/A | [session-persistence](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/session-persistence) | +| *nginx.org/keepalive* | *keepalive* | Sets the value of the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. Note that ``proxy_set_header Connection "";* is added to the generated configuration when the value > 0. | *0* | | +| *nginx.com/health-checks* | N/A | Enables active health checks. | *False* | [health-checks](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/health-checks) | +| *nginx.com/health-checks-mandatory* | N/A | Configures active health checks as mandatory. | *False* | [health-checks](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/health-checks) | +| *nginx.com/health-checks-mandatory-queue* | N/A | When active health checks are mandatory, creates a queue where incoming requests are temporarily stored while NGINX Plus is checking the health of the endpoints after a configuration reload. | *0* | [health-checks](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/health-checks) | +| *nginx.com/slow-start* | N/A | Sets the upstream server [slow-start period](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#server-slow-start). By default, slow-start is activated after a server becomes [available](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#passive-health-checks) or [healthy](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#active-health-checks). To enable slow-start for newly-added servers, configure [mandatory active health checks](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/health-checks). | *"0s"* | | +| *nginx.org/use-cluster-ip* | N/A | Enables using the Cluster IP and port of the service instead of the default behavior of using the IP and port of the pods. When this field is enabled, the fields that configure NGINX behavior related to multiple upstream servers (like ``lb-method* and ``next-upstream``) will have no effect, as NGINX Ingress Controller will configure NGINX with only one upstream server that will match the service Cluster IP. | *False* | | +{{}} + +### Rate limiting + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/limit-req-rate* | N/A | Enables request-rate-limiting for this ingress by creating a [limit_req_zone](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone) and matching [limit_req](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) for each location. All servers/locations of one ingress share the same zone. Must have unit r/s or r/m. | N/A | 200r/s | +| *nginx.org/limit-req-key* | N/A | The key to which the rate limit is applied. Can contain text, variables, or a combination of them. Variables must be surrounded by ${}. | ${binary_remote_addr} | ${binary_remote_addr} | +| *nginx.org/limit-req-zone-size* | N/A | Configures the size of the created [limit_req_zone](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone). | 10m | 20m | +| *nginx.org/limit-req-delay* | N/A | Configures the delay-parameter of the [limit_req](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) directive. | 0 | 100 | +| *nginx.org/limit-req-no-delay* | N/A | Configures the nodelay-parameter of the [limit_req](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) directive. | false | true | +| *nginx.org/limit-req-burst* | N/A | Configures the burst-parameter of the [limit_req](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) directive. | N/A | 100 | +| *nginx.org/limit-req-dry-run* | N/A | Enables the dry run mode. In this mode, the rate limit is not actually applied, but the number of excessive requests is accounted as usual in the shared memory zone. | false | true | +| *nginx.org/limit-req-log-level* | N/A | Sets the desired logging level for cases when the server refuses to process requests due to rate exceeding, or delays request processing. Allowed values are info, notice, warn or error. | error | info | +| *nginx.org/limit-req-reject-code* | N/A | Sets the status code to return in response to rejected requests. Must fall into the range 400..599. | 429 | 503 | +| *nginx.org/limit-req-scale* | N/A | Enables a constant rate-limit by dividing the configured rate by the number of nginx-ingress pods currently serving traffic. This adjustment ensures that the rate-limit remains consistent, even as the number of nginx-pods fluctuates due to autoscaling. Note: This will not work properly if requests from a client are not evenly distributed accross all ingress pods (sticky sessions, long lived TCP-Connections with many requests etc.). In such cases using [zone-sync]({{< ref "/configuration/global-configuration/configmap-resource.md#zone-sync" >}}) instead would give better results. Enabling `zone-sync` will suppress this setting. | false | true | +{{}} + +### Snippets and custom templates + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/location-snippets* | *location-snippets* | Sets a custom snippet in location context. | N/A | | +| *nginx.org/server-snippets* | *server-snippets* | Sets a custom snippet in server context. | N/A | | +{{}} + +### App Protect WAF {#app-protect} + +{{< note >}} The App Protect annotations only work if the App Protect WAF module is [installed]({{< relref "installation/integrations/app-protect-waf/installation.md" >}}). {{< /note >}} + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *appprotect.f5.com/app-protect-policy* | N/A | The name of the App Protect Policy for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace of the Ingress Resource is used. If not specified but ``appprotect.f5.com/app-protect-enable* is true, a default policy id applied. If the referenced policy resource does not exist, or policy is invalid, this annotation will be ignored, and the default policy will be applied. | N/A | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +| *appprotect.f5.com/app-protect-enable* | N/A | Enable App Protect for the Ingress Resource. | *False* | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +| *appprotect.f5.com/app-protect-security-log-enable* | N/A | Enable the [security log](/nginx-app-protect/troubleshooting/#app-protect-logging-overview) for App Protect. | *False* | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +| *appprotect.f5.com/app-protect-security-log* | N/A | The App Protect log configuration for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace as the Ingress Resource is used. If not specified the default is used which is: filter: ``illegal``, format: ``default``. Multiple configurations can be specified in a comma separated list. Both log configurations and destinations list (see below) must be of equal length. Configs and destinations are paired by the list indices. | N/A | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +| *appprotect.f5.com/app-protect-security-log-destination* | N/A | The destination of the security log. For more information check the [DESTINATION argument](/nginx-app-protect/troubleshooting/#app-protect-logging-overview). Multiple destinations can be specified in a comma-separated list. Both log configurations and destinations list (see above) must be of equal length. Configs and destinations are paired by the list indices. | *syslog:server=localhost:514* | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +{{}} + +### App Protect DoS + +{{< note >}} The App Protect DoS annotations only work if the App Protect DoS module is [installed]({{< relref "installation/integrations/app-protect-dos/installation.md" >}}). {{< /note >}} + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *appprotectdos.f5.com/app-protect-dos-resource* | N/A | Enable App Protect DoS for the Ingress Resource by specifying a [DosProtectedResource]({{< relref "installation/integrations/app-protect-dos/dos-protected.md" >}}). | N/A | [app-protect-dos](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-dos) | +{{}} diff --git a/content/includes/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md b/content/includes/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md new file mode 100644 index 000000000..642d041ae --- /dev/null +++ b/content/includes/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md @@ -0,0 +1,128 @@ +--- +docs: DOCS-592 +doctypes: +- '' +title: Advanced configuration with Snippets +toc: true +weight: 400 +--- + +Snippets allow you to insert raw NGINX config into different contexts of the NGINX configurations that F5 NGINX Ingress Controller generates. + +Snippets are intended for advanced NGINX users who need more control over the generated NGINX configuration, and can be used in cases where Annotations and ConfigMap entries would not apply. + + + +## Disadvantages of snippets + +Snippets are configured [using Annotations]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-annotations.md#snippets-and-custom-templates" >}}), but are disabled by default due to their complexity. They are also available through the [ConfigMap]({{< relref "/configuration/global-configuration/configmap-resource.md#snippets-and-custom-templates" >}}) resource. + +To use snippets, set the [`enable-snippets`]({{< ref "/nic/configuration/global-configuration/command-line-arguments.md#cmdoption-enable-snippets" >}}) command-line argument. + +Snippets have the following disadvantages: + +- *Complexity*. Snippets require you to: + - Understand NGINX configuration primitives and implement a correct NGINX configuration. + - Understand how NGINX Ingress Controller generates NGINX configuration so that a snippet doesn't interfere with the other features in the configuration. +- *Decreased robustness*. An incorrect snippet can invalidate NGINX configuration, causing reload failures. Until the snippet is fixed, it will prevent any new configuration updates, including updates for the other Ingress resources. +- *Security implications*. Snippets give access to NGINX configuration primitives, which are not validated by NGINX Ingress Controller. For example, a snippet can configure NGINX to serve the TLS certificates and keys used for TLS termination for Ingress resources. + +{{< note >}} If the NGINX configuration includes an invalid snippet, NGINX will continue to operate with the last valid configuration. {{< /note >}} + +## Using snippets + +The example below shows how to use snippets to customize the NGINX configuration template using annotations. + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: cafe-ingress-with-snippets + annotations: + nginx.org/server-snippets: | + location / { + return 302 /coffee; + } + nginx.org/location-snippets: | + add_header my-test-header test-value; +spec: + rules: + - host: cafe.example.com + http: + paths: + - path: /tea + pathType: Prefix + backend: + service: + name: tea-svc + port: + number: 80 + - path: /coffee + pathType: Prefix + backend: + service: + name: coffee-svc + port: + number: 80 +``` + +These snippets generate the following NGINX configuration: + +{{< note >}} The example is shortened for conciseness. {{< /note >}} + +```nginx +server { + listen 80; + + + location / { + return 302 /coffee; + } + + + location /coffee { + proxy_http_version 1.1; + + + add_header my-test-header test-value; + ... + proxy_pass http://default-cafe-ingress-with-snippets-cafe.example.com-coffee-svc-80; + } + + location /tea { + proxy_http_version 1.1; + + add_header my-test-header test-value; + ... + proxy_pass http://default-cafe-ingress-with-snippets-cafe.example.com-tea-svc-80; + } +} +``` + +## Troubleshooting + +If a snippet includes an invalid NGINX configuration, NGINX Ingress Controller will fail to reload NGINX. The error will be reported in NGINX Ingress Controller logs and an event with the error will be associated with the Ingress resource: + +An example of an error from the logs: + +```text +[emerg] 31#31: unknown directive "badd_header" in /etc/nginx/conf.d/default-cafe-ingress-with-snippets.conf:54 +Event(v1.ObjectReference{Kind:"Ingress", Namespace:"default", Name:"cafe-ingress-with-snippets", UID:"f9656dc9-63a6-41dd-a499-525b0e0309bb", APIVersion:"extensions/v1beta1", ResourceVersion:"2322030", FieldPath:""}): type: 'Warning' reason: 'AddedOrUpdatedWithError' Configuration for default/cafe-ingress-with-snippets was added or updated, but not applied: Error reloading NGINX for default/cafe-ingress-with-snippets: nginx reload failed: Command /usr/sbin/nginx -s reload stdout: "" +stderr: "nginx: [emerg] unknown directive \"badd_header\" in /etc/nginx/conf.d/default-cafe-ingress-with-snippets.conf:54\n" +finished with error: exit status 1 +``` + +An example of an event with an error (you can view events associated with the Ingress by running `kubectl describe -n nginx-ingress ingress nginx-ingress`): + +```text +Events: +Type Reason Age From Message +---- ------ ---- ---- ------- +Normal AddedOrUpdated 52m (x3 over 61m) nginx-ingress-controller Configuration for default/cafe-ingress-with-snippets was added or updated +finished with error: exit status 1 +Warning AddedOrUpdatedWithError 54s (x2 over 89s) nginx-ingress-controller Configuration for default/cafe-ingress-with-snippets was added or updated, but not applied: Error reloading NGINX for default/cafe-ingress-with-snippets: nginx reload failed: Command /usr/sbin/nginx -s reload stdout: "" +stderr: "nginx: [emerg] unknown directive \"badd_header\" in /etc/nginx/conf.d/default-cafe-ingress-with-snippets.conf:54\n" +finished with error: exit status 1 +``` + +Additionally, to help troubleshoot snippets, a number of Prometheus metrics show the stats about failed reloads – `controller_nginx_last_reload_status` and `controller_nginx_reload_errors_total`. diff --git a/content/includes/nic/configuration/ingress-resources/basic-configuration.md b/content/includes/nic/configuration/ingress-resources/basic-configuration.md new file mode 100644 index 000000000..bd2c10d26 --- /dev/null +++ b/content/includes/nic/configuration/ingress-resources/basic-configuration.md @@ -0,0 +1,107 @@ +--- +title: Basic configuration +weight: 100 +toc: true +type: reference +product: NIC +docs: DOCS-593 +--- + +This document shows a basic Ingress resource definition for F5 NGINX Ingress Controller. It load balances requests for two services as part of a single application. + +{{< ghcode `https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/ingress-resources/complete-example/cafe-ingress.yaml`>}} + +Here is a breakdown of what this Ingress resource definition means: + +- The `metadata.name` field defines the name of the resource `cafe‑ingress`. +- The `spec.tls` field sets up SSL/TLS termination: + - The `hosts` field applies the certificate and key to the `cafe.example.com` host. + - The `secretName` references a secret resource by its name, `cafe‑secret`. The secret must belong to the same namespace as the Ingress, of the type ``kubernetes.io/tls`` and contain keys named ``tls.crt`` and ``tls.key`` that hold the certificate and private key as described [here](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls>). If the secret doesn't exist or is invalid, NGINX will break any attempt to establish a TLS connection to the hosts to which the secret is applied. +- The `spec.rules` field defines a host with the domain name `cafe.example.com`. +- The `paths` field defines two path‑based rules: + - The rule with the path `/tea` instructs NGINX to distribute the requests with the `/tea` URI among the pods of the *tea* service, which is deployed with the name `tea‑svc` in the cluster. + - The rule with the path `/coffee` instructs NGINX to distribute the requests with the `/coffee` URI among the pods of the *coffee* service, which is deployed with the name `coffee‑svc` in the cluster. + - Both rules instruct NGINX to distribute the requests to `port 80` of the corresponding service (the `servicePort` field). + +To learn more about the Ingress resource, view [the official Kubernetes documentation for Ingress resources](https://kubernetes.io/docs/concepts/services-networking/ingress/). + +{{< note >}} For complete instructions on deploying Ingress and Secret resources in the cluster, see the [complete example](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/complete-example) in the GitHub repository. {{< /note >}} + +--- + +## New features available in Kubernetes 1.18 + +Starting from Kubernetes 1.18, you can use the following new features: + +- The host field supports wildcard domain names, such as `*.example.com`. +- The path supports different matching rules with the new field `pathType`, which takes the following values: `Prefix` for prefix-based matching, `Exact` for exact matching and `ImplementationSpecific`, which is the default type and is the same as `Prefix`. For example: + + ```yaml {hl_lines=[2, 7, 14]} + - path: /tea + pathType: Prefix + backend: + serviceName: tea-svc + servicePort: 80 + - path: /tea/green + pathType: Exact + backend: + service: + name: tea-svc + port: + number: 80 + - path: /coffee + pathType: ImplementationSpecific + backend: + service: + name: coffee-svc + port: + number: 80 + ``` + +- The `ingressClassName` field is now supported: + + ```yaml {hl_lines=[6]} + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cafe-ingress + spec: + ingressClassName: nginx + tls: + - hosts: + - cafe.example.com + secretName: cafe-secret + rules: + - host: cafe.example.com + . . . + ``` + + When using this field you need to create the `IngressClass` resource with the corresponding `name`. View the [Create common resources]({{< relref "installation/installing-nic/installation-with-manifests.md#create-common-resources" >}}) section of the Installation with Manifests topic for more information. + +--- + +## Restrictions + +NGINX Ingress Controller imposes the following restrictions on Ingress resources: + +- When defining an Ingress resource, the `host` field is required. +- The `host` value needs to be unique among all Ingress and VirtualServer resources unless the Ingress resource is a [mergeable minion]({{< ref "/nic/configuration/ingress-resources/cross-namespace-configuration.md" >}}). View the [Host and Listener collisions]({{< ref "/nic/configuration/host-and-listener-collisions.md" >}}) topic for more information. +- The `path` field in `spec.rules[].http.paths[]` is required for `Exact` and `Prefix` `pathTypes`. +- The ImplementationSpecific `pathType` is treated as equivalent to `Prefix` `pathType`, with the exception that when this `pathType` is configured, the `path` field in `spec.rules[].http.paths[]` is not mandatory. `path` defaults to `/` if not set but the `pathType` is set to ImplementationSpecific. + +--- + +## Advanced configuration + +NGINX Ingress Controller generates NGINX configuration by executing a template file that contains the configuration options. + +These options are set with the Ingress resource and NGINX Ingress Controller's ConfigMap. + +The Ingress resource only allows you to use basic NGINX features: host and path-based routing and TLS termination. + +For advanced configuration, you have two options: + +- [Annotations]({{< ref "/configuration/ingress-resources/advanced-configuration-with-annotations.md" >}}) can be used to rewrite request URIs or inserting additional response headers. +- [Snippets]({{< ref "/configuration/ingress-resources/advanced-configuration-with-snippets" >}}) can be used to insert raw NGINX configuration, changing generated files. + +Additionally, it is possible to customize the template, described in the [Custom templates]({{< relref "/configuration/global-configuration/custom-templates.md" >}}) topic. diff --git a/content/includes/nic/configuration/ingress-resources/cross-namespace-configuration.md b/content/includes/nic/configuration/ingress-resources/cross-namespace-configuration.md new file mode 100644 index 000000000..3e92b6403 --- /dev/null +++ b/content/includes/nic/configuration/ingress-resources/cross-namespace-configuration.md @@ -0,0 +1,14 @@ +--- +docs: DOCS-594 +doctypes: +- '' +title: Cross-namespace configuration +toc: true +weight: 500 +--- + +This topic explains how to spread Ingress configuration across different namespaces in F5 NGINX Ingress Controller. + +You can spread the Ingress configuration for a common host across multiple Ingress resources using Mergeable Ingress resources. Such resources can belong to the *same* or *different* namespaces. This enables easier management when using a large number of paths. See the [Mergeable Ingress Resources](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/mergeable-ingress-types) example in our GitHub repo. + +As an alternative to Mergeable Ingress resources, you can use [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/) for cross-namespace configuration. See the [Cross-Namespace Configuration](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/cross-namespace-configuration) example in our GitHub repo. diff --git a/content/includes/nic/configuration/ingress-resources/custom-annotations.md b/content/includes/nic/configuration/ingress-resources/custom-annotations.md new file mode 100644 index 000000000..8bbd136b0 --- /dev/null +++ b/content/includes/nic/configuration/ingress-resources/custom-annotations.md @@ -0,0 +1,146 @@ +--- +docs: DOCS-595 +doctypes: +- '' +title: Custom annotations +toc: true +weight: 300 +--- + +This topic explains how you can use custom annotations with F5 NGINX Ingress Controller. + +Custom annotations enable you to quickly extend the Ingress resource to support many advanced features of NGINX, such as rate limiting, caching, etc. + +## Overview + +NGINX Ingress Controller supports a number of annotations for the Ingress resource that fine tune NGINX configuration (for example, connection timeouts) or enable additional features (for example, JWT validation). The complete list of annotations is available [here](/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations). + +The annotations are provided only for the most common features and use cases, meaning that not every NGINX feature or a customization option is available through the annotations. Additionally, even if an annotation is available, it might not give you the satisfactory level of control of a particular NGINX feature. + +Custom annotations allow you to add an annotation for an NGINX feature that is not available as a regular annotation. In contrast with regular annotations, to add a custom annotation, you don't need to modify the Ingress Controller source code -- just modify the template. Additionally, with a custom annotation, you get full control of how the feature is implemented in NGINX configuration. + +## Usage + +The Ingress Controller generates NGINX configuration for Ingress resources by executing a configuration template. See [NGINX template](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/internal/configs/version1/nginx.ingress.tmpl) or [NGINX Plus template](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/internal/configs/version1/nginx-plus.ingress.tmpl). + +To support custom annotations, the template has access to the information about the Ingress resource - its *name*, *namespace* and *annotations*. It is possible to check if a particular annotation present in the Ingress resource and conditionally insert NGINX configuration directives at multiple NGINX contexts - `http`, `server`, `location` or `upstream`. Additionally, you can get the value that is set to the annotation. + +Consider the following excerpt from the template, which was extended to support two custom annotations: + +```jinja2 +# This is the configuration for {{$.Ingress.Name}}/{{$.Ingress.Namespace}} + +{{if index $.Ingress.Annotations "custom.nginx.org/feature-a"}} +# Insert config for feature A if the annotation is set +{{end}} + +{{with $value := index $.Ingress.Annotations "custom.nginx.org/feature-b"}} +# Insert config for feature B if the annotation is set +# Print the value assigned to the annotation: {{$value}} +{{end}} +``` + +Consider the following Ingress resource and note how we set two annotations: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: example-ingress + namespace: production + annotations: + custom.nginx.org/feature-a: "on" + custom.nginx.org/feature-b: "512" +spec: + rules: + - host: example.com + . . . +``` + +Assuming that the Ingress Controller is using that customized template, it will generate a config for the Ingress resource that will include the following part, generated by our template excerpt: + +```yaml +# This is the configuration for cafe-ingress/default + +# Insert config for feature A if the annotation is set + + + +# Insert config for feature B if the annotation is set +# Print the value assigned to the annotation: 512 +``` + +**Notes**: + +- You can customize the template to insert you custom annotations via [custom templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). +- The Ingress Controller uses go templates to generate NGINX config. You can read more information about go templates [here](https://golang.org/pkg/text/template/). + +See the examples in the next section that use custom annotations to configure NGINX features. + +### Custom Annotations with Mergeable Ingress Resources + +A Mergeable Ingress resource consists of multiple Ingress resources - one master and one or several minions. Read more about Mergeable Ingress resources [here](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration). + +If you'd like to use custom annotations with Mergeable Ingress resources, please keep the following in mind: + +- Custom annotations can be used in the Master and in Minions. For Minions, you can access them in the template only when processing locations. + + If you access `$.Ingress` anywhere in the Ingress template, you will get the master Ingress resource. To access a Minion Ingress resource, use `$location.MinionIngress`. However, it is only available when processing locations: + + ```jinja2 + {{range $location := $server.Locations}} + location {{$location.Path}} { + {{with $location.MinionIngress}} + # location for minion {{$location.MinionIngress.Namespace}}/{{$location.MinionIngress.Name}} + {{end}} + } {{end}} + ``` + + **Note**: `$location.MinionIngress` is a pointer. When a regular Ingress resource is processed in the template, the value of the pointer is `nil`. Thus, it is important that you check that `$location.MinionIngress` is not `nil` as in the example above using the `with` action. + +- Minions do not inherent custom annotations of the master. + +### Helper Functions + +Helper functions can be used in the Ingress template to parse the values of custom annotations. + +{{% table %}} +| Function | Input Arguments | Return Arguments | Description | +| ---| ---| ---| --- | +| ``split`` | ``s, sep string`` | ``[]string`` | Splits the string ``s`` into a slice of strings separated by the ``sep``. | +| ``trim`` | ``s string`` | ``string`` | Trims the trailing and leading whitespace from the string ``s``. | +| ``contains`` | ``s, substr string`` | ``bool`` | Tests whether the string ``substr`` is a substring of the string ``s``. | +| ``hasPrefix`` | ``s, prefix string`` | ``bool`` | Tests whether the string ``prefix`` is a prefix of the string ``s``. | +| ``hasSuffix`` | ``s, suffix string`` | ``bool`` | Tests whether the string ``suffix`` is a suffix of the string ``s``. | +| ``toLower`` | ``s string`` | ``bool`` | Converts all letters in the string ``s`` to their lower case. | +| ``toUpper`` | ``s string`` | ``bool`` | Converts all letters in the string ``s`` to their upper case. | +| ``replaceAll`` | ``s, old, new string`` | ``string`` | Replaces all occurrences of ``old`` with ``new`` in the string ``s``. | +{{% /table %}} + +Consider the following custom annotation `custom.nginx.org/allowed-ips`, which expects a comma-separated list of IP addresses: + +```yaml +annotations: + custom.nginx.org/allowed-ips: "192.168.1.3, 10.0.0.13" +``` + + The helper functions can parse the value of the `custom.nginx.org/allowed-ips` annotation, so that in the template you can use each IP address separately. Consider the following template excerpt: + +```jinja2 +{{range $ip := split (index $.Ingress.Annotations "custom.nginx.org/allowed-ips") ","}} + allow {{trim $ip}}; +{{end}} +deny all; +``` + +The template excerpt will generate the following configuration: + +``` +allow 192.168.1.3; +allow 10.0.0.13; +deny all; +``` + +## Example + +See the [custom annotations example](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/examples/ingress-resources/custom-annotations). diff --git a/content/includes/nic/configuration/policy-resource.md b/content/includes/nic/configuration/policy-resource.md new file mode 100644 index 000000000..6ea2c8121 --- /dev/null +++ b/content/includes/nic/configuration/policy-resource.md @@ -0,0 +1,945 @@ +--- +title: Policy resources +weight: 500 +toc: true +type: how-to +product: NIC +docs: DOCS-596 +--- + +The Policy resource allows you to configure features like access control and rate-limiting, which you can add to your [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/). + +The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + +This document is the reference documentation for the Policy resource. An example of a Policy for access control is available in our [GitHub repository](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/examples/custom-resources/access-control). + +## Prerequisites + +Policies work together with [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/), which you need to create separately. + +## Policy Specification + +Below is an example of a policy that allows access for clients from the subnet `10.0.0.0/8` and denies access for any other clients: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: Policy +metadata: + name: allow-localhost +spec: + accessControl: + allow: + - 10.0.0.0/8 +``` + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``accessControl`` | The access control policy based on the client IP address. | [accessControl](#accesscontrol) | No | +|``ingressClassName`` | Specifies which instance of NGINX Ingress Controller must handle the Policy resource. | ``string`` | No | +|``rateLimit`` | The rate limit policy controls the rate of processing requests per a defined key. | [rateLimit](#ratelimit) | No | +|``apiKey`` | The API Key policy configures NGINX to authorize requests which provide a valid API Key in a specified header or query param. | [apiKey](#apikey) | No | +|``basicAuth`` | The basic auth policy configures NGINX to authenticate client requests using HTTP Basic authentication credentials. | [basicAuth](#basicauth) | No | +|``jwt`` | The JWT policy configures NGINX Plus to authenticate client requests using JSON Web Tokens. | [jwt](#jwt) | No | +|``ingressMTLS`` | The IngressMTLS policy configures client certificate verification. | [ingressMTLS](#ingressmtls) | No | +|``egressMTLS`` | The EgressMTLS policy configures upstreams authentication and certificate verification. | [egressMTLS](#egressmtls) | No | +|``waf`` | The WAF policy configures WAF and log configuration policies for [NGINX AppProtect]({{< relref "installation/integrations/app-protect-waf/configuration.md" >}}) | [WAF](#waf) | No | +{{% /table %}} + +\* A policy must include exactly one policy. + +### AccessControl + +The access control policy configures NGINX to deny or allow requests from clients with the specified IP addresses/subnets. + +For example, the following policy allows access for clients from the subnet `10.0.0.0/8` and denies access for any other clients: + +```yaml +accessControl: + allow: + - 10.0.0.0/8 +``` + +In contrast, the policy below does the opposite: denies access for clients from `10.0.0.0/8` and allows access for any other clients: + +```yaml +accessControl: + deny: + - 10.0.0.0/8 +``` +{{< note >}} + +The feature is implemented using the NGINX [ngx_http_access_module](http://nginx.org/en/docs/http/ngx_http_access_module.html). NGINX Ingress Controller access control policy supports either allow or deny rules, but not both (as the module does). + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``allow`` | Allows access for the specified networks or addresses. For example, ``192.168.1.1`` or ``10.1.1.0/16``. | ``[]string`` | No | +|``deny`` | Denies access for the specified networks or addresses. For example, ``192.168.1.1`` or ``10.1.1.0/16``. | ``[]string`` | No | \* an accessControl must include either `allow` or `deny`. | +{{% /table %}} + +#### AccessControl Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple access control policies. For example, here we reference two policies, each with configured allow lists: + +```yaml +policies: +- name: allow-policy-one +- name: allow-policy-two +``` + +When you reference more than one access control policy, NGINX Ingress Controller will merge the contents into a single allow list or a single deny list. + +Referencing both allow and deny policies, as shown in the example below, is not supported. If both allow and deny lists are referenced, NGINX Ingress Controller uses just the allow list policies. + +```yaml +policies: +- name: deny-policy +- name: allow-policy-one +- name: allow-policy-two +``` + +### RateLimit + +The rate limit policy configures NGINX to limit the processing rate of requests. + +For example, the following policy will limit all subsequent requests coming from a single IP address once a rate of 10 requests per second is exceeded: + +```yaml +rateLimit: + rate: 10r/s + zoneSize: 10M + key: ${binary_remote_addr} +``` +{{< note >}} + +The feature is implemented using the NGINX [ngx_http_limit_req_module](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html). + +{{< /note >}} + +{{< note >}} + +When the [Zone Sync feature]({{< ref "/configuration/global-configuration/configmap-resource.md#zone-sync" >}}) is enabled with NGINX Plus, the rate limiting zone will be synchronized across all replicas in the cluster. This means all replicas are aware of the requests that have been rate limited by other replicas in the cluster. + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``rate`` | The rate of requests permitted. The rate is specified in requests per second (r/s) or requests per minute (r/m). | ``string`` | Yes | +|``key`` | The key to which the rate limit is applied. Can contain text, variables, or a combination of them. Variables must be surrounded by ``${}``. For example: ``${binary_remote_addr}``. Accepted variables are ``$binary_remote_addr``, ``$request_uri``,``$request_method``, ``$url``, ``$http_``, ``$args``, ``$arg_``, ``$cookie_``, ``$jwt_claim_``. | ``string`` | Yes | +|``zoneSize`` | Size of the shared memory zone. Only positive values are allowed. Allowed suffixes are ``k`` or ``m``, if none are present ``k`` is assumed. | ``string`` | Yes | +|``delay`` | The delay parameter specifies a limit at which excessive requests become delayed. If not set all excessive requests are delayed. | ``int`` | No | +|``noDelay`` | Disables the delaying of excessive requests while requests are being limited. Overrides ``delay`` if both are set. | ``bool`` | No | +|``burst`` | Excessive requests are delayed until their number exceeds the ``burst`` size, in which case the request is terminated with an error. | ``int`` | No | +|``dryRun`` | Enables the dry run mode. In this mode, the rate limit is not actually applied, but the number of excessive requests is accounted as usual in the shared memory zone. | ``bool`` | No | +|``logLevel`` | Sets the desired logging level for cases when the server refuses to process requests due to rate exceeding, or delays request processing. Allowed values are ``info``, ``notice``, ``warn`` or ``error``. Default is ``error``. | ``string`` | No | +|``rejectCode`` | Sets the status code to return in response to rejected requests. Must fall into the range ``400..599``. Default is ``503``. | ``int`` | No | +|``scale`` | Enables a constant rate-limit by dividing the configured rate by the number of nginx-ingress pods currently serving traffic. This adjustment ensures that the rate-limit remains consistent, even as the number of nginx-pods fluctuates due to autoscaling. **This will not work properly if requests from a client are not evenly distributed across all ingress pods** (Such as with sticky sessions, long lived TCP Connections with many requests, and so forth). In such cases using [zone-sync]({{< ref "/configuration/global-configuration/configmap-resource.md#zone-sync" >}}) instead would give better results. Enabling `zone-sync` will suppress this setting. | ``bool`` | No | +|``condition`` | Add a condition to a rate-limit policy. | [ratelimit.condition](#ratelimitcondition) | No | +{{% /table %}} + +{{< note >}} + +For each policy referenced in a VirtualServer and/or its VirtualServerRoutes, NGINX Ingress Controller will generate a single rate limiting zone defined by the [`limit_req_zone`](http://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone) directive. If two VirtualServer resources reference the same policy, NGINX Ingress Controller will generate two different rate limiting zones, one zone per VirtualServer. + +{{< /note >}} + +#### RateLimit Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple rate limit policies. For example, here we reference two policies: + +```yaml +policies: +- name: rate-limit-policy-one +- name: rate-limit-policy-two +``` + +When you reference more than one rate limit policy, NGINX Ingress Controller will configure NGINX to use all referenced rate limits. When you define multiple policies, each additional policy inherits the `dryRun`, `logLevel`, and `rejectCode` parameters from the first policy referenced (`rate-limit-policy-one`, in the example above). + +### RateLimit.Condition + +RateLimit.Condition defines a condition for a rate limit policy. For example: + +```yaml +condition: + jwt: + claim: user_details.level + match: premium + default: true +``` + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``jwt`` | defines a JWT condition to rate limit against. | [ratelimit.condition.jwt](#ratelimitconditionjwt) | No | +|``default`` | sets the rate limit in this policy to be the default if no conditions are met. In a group of policies with the same JWT condition, only one policy can be the default. | ``bool`` | No | +{{% /table %}} + +The rate limit policy with condition is designed to be used in combination with one or more rate limit policies. For example, multiple rate limit policies with [RateLimit.Condition.JWT](#ratelimitconditionjwt) can be used to apply different tiers of rate limit based on the value of a JWT claim. For a practical example of tiered rate limiting by the value of a JWT claim, see the example in our [GitHub repository](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/rate-limit-tiered-jwt-claim/README.md). + +### RateLimit.Condition.JWT +{{< note >}} + +This feature is only available with NGINX Plus. + +{{< /note >}} + +RateLimit.Condition.JWT defines a condition for a rate limit by JWT claim. For example, here we define a condition for a rate limit policy that only applies to requests with a JWT claim `user_details.level` with a value `premium`: + +```yaml +jwt: + claim: user_details.level + match: premium +``` + +The rate limit policy will only apply to requests that contain a JWT with the specified claim and value. For example, the following JWT payload will match the JWT condition: + +```json +{ + "user_details": { + "level": "premium" + }, + "sub": "client1" +} +``` + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``claim`` | Claim is the JWT claim to be rate limit by. Nested claims should be separated by ".". | ``string`` | Yes | +|``match`` | the value of the claim to match against. | ``string`` | Yes | +{{% /table %}} + +### APIKey + +The API Key auth policy configures NGINX to authorize client requests based on the presence of a valid API Key in a header or query param specified in the policy. + +{{< note >}} + +The feature is implemented using NGINX [ngx_http_auth_request_module](http://nginx.org/en/docs/http/ngx_http_auth_request_module.html) and [NGINX JavaScript (NJS)](https://nginx.org/en/docs/njs/). + +{{< /note >}} + +The policies' API keys are securely stored using SHA-256 hashing. When a client sends an API Key, it is hashed by NJS and then compared to the hashed API Key in the NGINX config. + +If the hashed keys match, the NGINX JavaScript (NJS) subrequest issues a 204 No Content response to the `auth_request` directive, indicating successful authorization. Conversely, if no API Key is provided in the specified header or query parameter, a 401 Unauthorized response is returned. Similarly, if an invalid key is presented in the expected header or query parameter, a 403 Forbidden response is issued, denying access. + +It is possible to use the [errorPages](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#errorpage) property on a route, to change the default behaviour of 401 or 403 errors. + +At least one header or query param is required. + +The policy below configures NGINX Ingress Controller to require the API Key `password` in the header "my-header". + +```yaml +apiKey: + suppliedIn: + header: + - "my-header" + clientSecret: api-key-secret +``` + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: api-key-secret +type: nginx.org/apikey +data: + client1: cGFzc3dvcmQ= # password +``` + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``suppliedIn`` | `header` or `query`. | | Yes | +|``suppliedIn.header`` | An array of headers that the API Key may appear in. | ``string[]`` | No | +|``suppliedIn.query`` | An array of query params that the API Key may appear in. | ``string[]`` | No | +|``clientSecret`` | The name of the Kubernetes secret that stores the API Key(s). It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/apikey``, and the API Key(s) must be stored in a key: val format where each key is a unique clientID and each value is a unique base64 encoded API Key | ``string`` | Yes | +{{% /table %}} + +{{}}An APIKey Policy must include a minimum of one of the `suppliedIn.header` or `suppliedIn.query` parameters. Both can also be supplied.{{}} + +#### APIKey Merging Behavior + +A VirtualServer or VirtualServerRoute can be associated with only one API Key policy per route or subroute. However, it is possible to replace an API Key policy from a higher-level with a different policy defined on a more specific route. + +For example, a VirtualServer can implement different API Key policies at various levels. In the configuration below, the server-wide api-key-policy-server applies to /backend1 for authorization, as it lacks a more specific policy. Meanwhile, /backend2 uses the api-key-policy-route defined at the route level. + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server +spec: + host: virtual-server.example.com + policies: + - name: api-key-policy-server + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1 + service: backend1-svc + port: 80 + routes: + - path: /backend1 + action: + pass: backend1 + - path: /backend2 + action: + pass: backend2 + policies: + - name: api-key-policy-route +``` + +### BasicAuth + +The basic auth policy configures NGINX to authenticate client requests using the [HTTP Basic authentication scheme](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication). + +For example, the following policy will reject all requests that do not include a valid username/password combination in the HTTP header `Authentication` + +```yaml +basicAuth: + secret: htpasswd-secret + realm: "My API" +``` +{{< note >}} +The feature is implemented using the NGINX [ngx_http_auth_basic_module](https://nginx.org/en/docs/http/ngx_http_auth_basic_module.html). +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``secret`` | The name of the Kubernetes secret that stores the Htpasswd configuration. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/htpasswd``, and the config must be stored in the secret under the key ``htpasswd``, otherwise the secret will be rejected as invalid. | ``string`` | Yes | +|``realm`` | The realm for the basic authentication. | ``string`` | No | +{{% /table %}} + +#### BasicAuth Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple basic auth policies. However, only one can be applied. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: basic-auth-policy-one +- name: basic-auth-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `basic-auth-policy-one`, and ignores `basic-auth-policy-two`. + +### JWT Using Local Kubernetes Secret + +{{< note >}} + +This feature is only available with NGINX Plus. + +{{< /note >}} + +The JWT policy configures NGINX Plus to authenticate client requests using JSON Web Tokens. + +The following example policy will reject all requests that do not include a valid JWT in the HTTP header `token`: + +```yaml +jwt: + secret: jwk-secret + realm: "My API" + token: $http_token +``` + +You can pass the JWT claims and JOSE headers to the upstream servers. For example: + +```yaml +action: + proxy: + upstream: webapp + requestHeaders: + set: + - name: user + value: ${jwt_claim_user} + - name: alg + value: ${jwt_header_alg} +``` + +We use the `requestHeaders` of the [Action.Proxy](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#actionproxy) to set the values of two headers that NGINX will pass to the upstream servers. + +The value of the `${jwt_claim_user}` variable is the `user` claim of a JWT. For other claims, use `${jwt_claim_name}`, where `name` is the name of the claim. Note that nested claims and claims that include a period (`.`) are not supported. Similarly, use `${jwt_header_name}` where `name` is the name of a header. In our example, we use the `alg` header. + +{{< note >}} + +This feature is implemented using the NGINX Plus [ngx_http_auth_jwt_module](https://nginx.org/en/docs/http/ngx_http_auth_jwt_module.html). + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``secret`` | The name of the Kubernetes secret that stores the JWK. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/jwk``, and the JWK must be stored in the secret under the key ``jwk``, otherwise the secret will be rejected as invalid. | ``string`` | Yes | +|``realm`` | The realm of the JWT. | ``string`` | Yes | +|``token`` | The token specifies a variable that contains the JSON Web Token. By default the JWT is passed in the ``Authorization`` header as a Bearer Token. JWT may be also passed as a cookie or a part of a query string, for example: ``$cookie_auth_token``. Accepted variables are ``$http_``, ``$arg_``, ``$cookie_``. | ``string`` | No | +{{% /table %}} + +#### JWT Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple JWT policies. However, only one can be applied: every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: jwt-policy-one +- name: jwt-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `jwt-policy-one`, and ignores `jwt-policy-two`. + +### JWT Using JWKS From Remote Location + +{{< note >}} + +This feature is only available with NGINX Plus. + +{{< /note >}} + +The JWT policy configures NGINX Plus to authenticate client requests using JSON Web Tokens, allowing import of the keys (JWKS) for JWT policy by means of a URL (for a remote server or an identity provider) as a result they don't have to be copied and updated to the IC pod. + +The following example policy will reject all requests that do not include a valid JWT in the HTTP header fetched from the identity provider: + +```yaml +jwt: + realm: MyProductAPI + token: $http_token + jwksURI: + keyCache: 1h +``` + +{{< note >}} + +This feature is implemented using the NGINX Plus directive [auth_jwt_key_request](http://nginx.org/en/docs/http/ngx_http_auth_jwt_module.html#auth_jwt_key_request) under [ngx_http_auth_jwt_module](https://nginx.org/en/docs/http/ngx_http_auth_jwt_module.html). + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``jwksURI`` | The remote URI where the request will be sent to retrieve JSON Web Key set| ``string`` | Yes | +|``keyCache`` | Enables in-memory caching of JWKS (JSON Web Key Sets) that are obtained from the ``jwksURI`` and sets a valid time for expiration. | ``string`` | Yes | +|``realm`` | The realm of the JWT. | ``string`` | Yes | +|``token`` | The token specifies a variable that contains the JSON Web Token. By default the JWT is passed in the ``Authorization`` header as a Bearer Token. JWT may be also passed as a cookie or a part of a query string, for example: ``$cookie_auth_token``. Accepted variables are ``$http_``, ``$arg_``, ``$cookie_``. | ``string`` | No | +{{% /table %}} + +{{< note >}} + +Content caching is enabled by default for each JWT policy with a default time of 12 hours. + +This is done to ensure to improve resiliency by allowing the JWKS (JSON Web Key Set) to be retrieved from the cache even when it has expired. + +{{< /note >}} + +#### JWT Merging Behavior + +This behavior is similar to using a local Kubernetes secret where a VirtualServer/VirtualServerRoute can reference multiple JWT policies. However, only one can be applied: every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: jwt-policy-one +- name: jwt-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `jwt-policy-one`, and ignores `jwt-policy-two`. + +### IngressMTLS + +The IngressMTLS policy configures client certificate verification. + +For example, the following policy will verify a client certificate using the CA certificate specified in the `ingress-mtls-secret`: + +```yaml +ingressMTLS: + clientCertSecret: ingress-mtls-secret + verifyClient: "on" + verifyDepth: 1 +``` + +Below is an example of the `ingress-mtls-secret` using the secret type `nginx.org/ca` + +```yaml +kind: Secret +metadata: + name: ingress-mtls-secret +apiVersion: v1 +type: nginx.org/ca +data: + ca.crt: +``` + +A VirtualServer that references an IngressMTLS policy must: + +- Enable [TLS termination](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#virtualservertls). +- Reference the policy in the VirtualServer [`spec`](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#virtualserver-specification). It is not allowed to reference an IngressMTLS policy in a [`route`](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#virtualserverroute) or in a VirtualServerRoute [`subroute`](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#virtualserverroutesubroute). + +If the conditions above are not met, NGINX will send the `500` status code to clients. + +You can pass the client certificate details, including the certificate, to the upstream servers. For example: + +```yaml +action: + proxy: + upstream: webapp + requestHeaders: + set: + - name: client-cert-subj-dn + value: ${ssl_client_s_dn} # subject DN + - name: client-cert + value: ${ssl_client_escaped_cert} # client certificate in the PEM format (urlencoded) +``` + +We use the `requestHeaders` of the [Action.Proxy](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#actionproxy) to set the values of the two headers that NGINX will pass to the upstream servers. See the [list of embedded variables](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#variables) that are supported by the `ngx_http_ssl_module`, which you can use to pass the client certificate details. + +{{< note >}} + + The feature is implemented using the NGINX [ngx_http_ssl_module](https://nginx.org/en/docs/http/ngx_http_ssl_module.html). + + {{< /note >}} + +#### Using a Certificate Revocation List + +The IngressMTLS policy supports configuring at CRL for your policy. +This can be done in one of two ways. + +{{< note >}} + + Only one of these configurations options can be used at a time. + +{{< /note >}} + +1. Adding the `ca.crl` field to the `nginx.org/ca` secret type, which accepts a base64 encoded certificate revocation list (crl). + Example YAML: + +```yaml +kind: Secret +metadata: + name: ingress-mtls-secret +apiVersion: v1 +type: nginx.org/ca +data: + ca.crt: + ca.crl: +``` + +2. Adding the `crlFileName` field to your IngressMTLS policy spec with the name of the CRL file. + +{{< note >}} + +This configuration option should only be used when using a CRL that is larger than 1MiB. + +Otherwise we recommend using the `nginx.org/ca` secret type for managing your CRL. + +{{< /note >}} + +Example YAML: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: Policy +metadata: + name: ingress-mtls-policy +spec: +ingressMTLS: + clientCertSecret: ingress-mtls-secret + crlFileName: webapp.crl + verifyClient: "on" + verifyDepth: 1 +``` + +**IMPORTANT NOTE** +When configuring a CRL with the `ingressMTLS.crlFileName` field, there is additional context to keep in mind: + +1. NGINX Ingress Controller will expect the CRL, in this case `webapp.crl`, will be in `/etc/nginx/secrets`. A volume mount will need to be added to NGINX Ingress Controller deployment add your CRL to `/etc/nginx/secrets` +2. When updating the content of your CRL (e.g a new certificate has been revoked), NGINX will need to be reloaded to pick up the latest changes. Depending on your environment this may require updating the name of your CRL and applying this update to your `ingress-mtls.yaml` policy to ensure NGINX picks up the latest CRL. + +Please refer to the Kubernetes documentation on [volumes](https://kubernetes.io/docs/concepts/storage/volumes/) to find the best implementation for your environment. + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``clientCertSecret`` | The name of the Kubernetes secret that stores the CA certificate. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/ca``, and the certificate must be stored in the secret under the key ``ca.crt``, otherwise the secret will be rejected as invalid. | ``string`` | Yes | +|``verifyClient`` | Verification for the client. Possible values are ``"on"``, ``"off"``, ``"optional"``, ``"optional_no_ca"``. The default is ``"on"``. | ``string`` | No | +|``verifyDepth`` | Sets the verification depth in the client certificates chain. The default is ``1``. | ``int`` | No | +|``crlFileName`` | The file name of the Certificate Revocation List. NGINX Ingress Controller will look for this file in `/etc/nginx/secrets` | ``string`` | No | +{{% /table %}} + +#### IngressMTLS Merging Behavior + +A VirtualServer can reference only a single IngressMTLS policy. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: ingress-mtls-policy-one +- name: ingress-mtls-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `ingress-mtls-policy-one`, and ignores `ingress-mtls-policy-two`. + +### EgressMTLS + +The EgressMTLS policy configures upstreams authentication and certificate verification. + +For example, the following policy will use `egress-mtls-secret` to authenticate with the upstream application and `egress-trusted-ca-secret` to verify the certificate of the application: + +```yaml +egressMTLS: + tlsSecret: egress-mtls-secret + trustedCertSecret: egress-trusted-ca-secret + verifyServer: on + verifyDepth: 2 +``` + +{{< note >}} + +The feature is implemented using the NGINX [ngx_http_proxy_module](https://nginx.org/en/docs/http/ngx_http_proxy_module.html). + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``tlsSecret`` | The name of the Kubernetes secret that stores the TLS certificate and key. It must be in the same namespace as the Policy resource. The secret must be of the type ``kubernetes.io/tls``, the certificate must be stored in the secret under the key ``tls.crt``, and the key must be stored under the key ``tls.key``, otherwise the secret will be rejected as invalid. | ``string`` | No | +|``trustedCertSecret`` | The name of the Kubernetes secret that stores the CA certificate. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/ca``, and the certificate must be stored in the secret under the key ``ca.crt``, otherwise the secret will be rejected as invalid. | ``string`` | No | +|``verifyServer`` | Enables verification of the upstream HTTPS server certificate. | ``bool`` | No | +|``verifyDepth`` | Sets the verification depth in the proxied HTTPS server certificates chain. The default is ``1``. | ``int`` | No | +|``sessionReuse`` | Enables reuse of SSL sessions to the upstreams. The default is ``true``. | ``bool`` | No | +|``serverName`` | Enables passing of the server name through ``Server Name Indication`` extension. | ``bool`` | No | +|``sslName`` | Allows overriding the server name used to verify the certificate of the upstream HTTPS server. | ``string`` | No | +|``ciphers`` | Specifies the enabled ciphers for requests to an upstream HTTPS server. The default is ``DEFAULT``. | ``string`` | No | +|``protocols`` | Specifies the protocols for requests to an upstream HTTPS server. The default is ``TLSv1 TLSv1.1 TLSv1.2``. | ``string`` | No | > Note: the value of ``ciphers`` and ``protocols`` is not validated by NGINX Ingress Controller. As a result, NGINX can fail to reload the configuration. To ensure that the configuration for a VirtualServer/VirtualServerRoute that references the policy was successfully applied, check its [status](/nginx-ingress-controller/configuration/global-configuration/reporting-resources-status/#virtualserver-and-virtualserverroute-resources). The validation will be added in the future releases. | +{{% /table %}} + +#### EgressMTLS Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple EgressMTLS policies. However, only one can be applied. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: egress-mtls-policy-one +- name: egress-mtls-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `egress-mtls-policy-one`, and ignores `egress-mtls-policy-two`. + +### OIDC + +{{< tip >}} + +This feature is disabled by default. To enable it, set the [enable-oidc]({{< ref "/nic/configuration/global-configuration/command-line-arguments.md#cmdoption-enable-oidc" >}}) command-line argument of NGINX Ingress Controller. + +{{< /tip >}} + +The OIDC policy configures NGINX Plus as a relying party for OpenID Connect authentication. + +For example, the following policy will use the client ID `nginx-plus` and the client secret `oidc-secret` to authenticate with the OpenID Connect provider `https://idp.example.com`: + +```yaml +spec: + oidc: + clientID: nginx-plus + clientSecret: oidc-secret + authEndpoint: https://idp.example.com/openid-connect/auth + tokenEndpoint: https://idp.example.com/openid-connect/token + jwksURI: https://idp.example.com/openid-connect/certs + endSessionEndpoint: https://idp.example.com/openid-connect/logout + postLogoutRedirectURI: / + accessTokenEnable: true +``` + +NGINX Plus will pass the ID of an authenticated user to the backend in the HTTP header `username`. + +{{< note >}} + +The feature is implemented using the [reference implementation](https://github.com/nginxinc/nginx-openid-connect/) of NGINX Plus as a relying party for OpenID Connect authentication. + +{{< /note >}} + +#### Prerequisites + +In order to use OIDC, you need to enable [zone synchronization](https://docs.nginx.com/nginx/admin-guide/high-availability/zone_sync/). If you don't set up zone synchronization, NGINX Plus will fail to reload. +You also need to configure a resolver, which NGINX Plus will use to resolve the IDP authorization endpoint. You can find an example configuration [in our GitHub repository](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/examples/custom-resources/oidc#step-7---configure-nginx-plus-zone-synchronization-and-resolver). + +{{< warning >}} + +The configuration in the example doesn't enable TLS and the synchronization between the replica happens in clear text. This could lead to the exposure of tokens. + +{{< /warning >}} + +#### Limitations + +The OIDC policy defines a few internal locations that can't be customized: `/_jwks_uri`, `/_token`, `/_refresh`, `/_id_token_validation`, `/logout`. In addition, as explained below, `/_codexch` is the default value for redirect URI, and `/_logout` is the default value for post logout redirect URI, both of which can be customized. Specifying one of these locations as a route in the VirtualServer or VirtualServerRoute will result in a collision and NGINX Plus will fail to reload. + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``clientID`` | The client ID provided by your OpenID Connect provider. | ``string`` | Yes | +|``clientSecret`` | The name of the Kubernetes secret that stores the client secret provided by your OpenID Connect provider. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/oidc``, and the secret under the key ``client-secret``, otherwise the secret will be rejected as invalid. | ``string`` | Yes | +|``authEndpoint`` | URL for the authorization endpoint provided by your OpenID Connect provider. | ``string`` | Yes | +|``authExtraArgs`` | A list of extra URL arguments to pass to the authorization endpoint provided by your OpenID Connect provider. Arguments must be URL encoded, multiple arguments may be included in the list, for example ``[ arg1=value1, arg2=value2 ]`` | ``string[]`` | No | +|``tokenEndpoint`` | URL for the token endpoint provided by your OpenID Connect provider. | ``string`` | Yes | +|``endSessionEndpoint`` | URL provided by your OpenID Connect provider to request the end user be logged out. | ``string`` | No | +|``jwksURI`` | URL for the JSON Web Key Set (JWK) document provided by your OpenID Connect provider. | ``string`` | Yes | +|``scope`` | List of OpenID Connect scopes. The scope ``openid`` always needs to be present and others can be added concatenating them with a ``+`` sign, for example ``openid+profile+email``, ``openid+email+userDefinedScope``. The default is ``openid``. | ``string`` | No | +|``redirectURI`` | Allows overriding the default redirect URI. The default is ``/_codexch``. | ``string`` | No | +|``postLogoutRedirectURI`` | URI to redirect to after the logout has been performed. Requires ``endSessionEndpoint``. The default is ``/_logout``. | ``string`` | No | +|``zoneSyncLeeway`` | Specifies the maximum timeout in milliseconds for synchronizing ID/access tokens and shared values between Ingress Controller pods. The default is ``200``. | ``int`` | No | +|``accessTokenEnable`` | Option of whether Bearer token is used to authorize NGINX to access protected backend. | ``boolean`` | No | +{{% /table %}} + +{{< note >}} + +Only one OIDC policy can be referenced in a VirtualServer and its VirtualServerRoutes. However, the same policy can still be applied to different routes in the VirtualServer and VirtualServerRoutes. + +{{< /note >}} + +#### OIDC Merging Behavior + +A VirtualServer/VirtualServerRoute can reference only a single OIDC policy. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: oidc-policy-one +- name: oidc-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `oidc-policy-one`, and ignores `oidc-policy-two`. + +## Using Policy + +You can use the usual `kubectl` commands to work with Policy resources, just as with built-in Kubernetes resources. + +For example, the following command creates a Policy resource defined in `access-control-policy-allow.yaml` with the name `webapp-policy`: + +```shell +kubectl apply -f access-control-policy-allow.yaml + +policy.k8s.nginx.org/webapp-policy configured +``` + +You can get the resource by running: + +```shell +kubectl get policy webapp-policy + +NAME AGE +webapp-policy 27m +``` + +For `kubectl get` and similar commands, you can also use the short name `pol` instead of `policy`. + +### WAF {#waf} + +{{< note >}} The feature is implemented using the NGINX Plus [NGINX App Protect WAF Module](https://docs.nginx.com/nginx-app-protect/configuration/). {{< /note >}} + +The WAF policy configures NGINX Plus to secure client requests using App Protect WAF policies. + +For example, the following policy will enable the referenced APPolicy. You can configure multiple APLogConfs with log destinations: + +```yaml +waf: + enable: true + apPolicy: "default/dataguard-alarm" + securityLogs: + - enable: true + apLogConf: "default/logconf" + logDest: "syslog:server=syslog-svc.default:514" + - enable: true + apLogConf: "default/logconf" + logDest: "syslog:server=syslog-svc-secondary.default:514" +``` + +{{< note >}} The field `waf.securityLog` is deprecated and will be removed in future releases.It will be ignored if `waf.securityLogs` is populated. {{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables NGINX App Protect WAF. | ``bool`` | Yes | +|``apPolicy`` | The [App Protect WAF policy]({{< relref "installation/integrations/app-protect-waf/configuration.md#waf-policies" >}}) of the WAF. Accepts an optional namespace. Mutually exclusive with ``apBundle``. | ``string`` | No | +|``apBundle`` | The [App Protect WAF policy bundle]({{< relref "installation/integrations/app-protect-waf/configuration.md#waf-bundles" >}}). Mutually exclusive with ``apPolicy``. | ``string`` | No | +|``securityLog.enable`` | Enables security log. | ``bool`` | No | +|``securityLog.apLogConf`` | The [App Protect WAF log conf]({{< relref "installation/integrations/app-protect-waf/configuration.md#waf-logs" >}}) resource. Accepts an optional namespace. Only works with ``apPolicy``. | ``string`` | No | +|``securityLog.apLogBundle`` | The [App Protect WAF log bundle]({{< relref "installation/integrations/app-protect-waf/configuration.md#waf-bundles" >}}) resource. Only works with ``apBundle``. | ``string`` | No | +|``securityLog.logDest`` | The log destination for the security log. Only accepted variables are ``syslog:server=:``, ``stderr``, ````. | ``string`` | No | +{{% /table %}} + +#### WAF Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple WAF policies. However, only one can be applied. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: waf-policy-one +- name: waf-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `waf-policy-one`, and ignores `waf-policy-two`. + +### Applying Policies + +You can apply policies to both VirtualServer and VirtualServerRoute resources. For example: + +- VirtualServer: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: VirtualServer + metadata: + name: cafe + namespace: cafe + spec: + host: cafe.example.com + tls: + secret: cafe-secret + policies: # spec policies + - name: policy1 + upstreams: + - name: coffee + service: coffee-svc + port: 80 + routes: + - path: /tea + policies: # route policies + - name: policy2 + namespace: cafe + route: tea/tea + - path: /coffee + policies: # route policies + - name: policy3 + namespace: cafe + action: + pass: coffee + ``` + + For VirtualServer, you can apply a policy: + * to all routes (spec policies) + * to a specific route (route policies) + + Route policies of the *same type* override spec policies. In the example above, if the type of the policies `policy-1` and `policy-3` is `accessControl`, then for requests to `cafe.example.com/coffee`, NGINX will apply `policy-3`. + + The overriding is enforced by NGINX: the spec policies are implemented in the `server` context of the config, and the route policies are implemented in the `location` context. As a result, the route policies of the same type win. + +- VirtualServerRoute, which is referenced by the VirtualServer above: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: VirtualServerRoute + metadata: + name: tea + namespace: tea + spec: + host: cafe.example.com + upstreams: + - name: tea + service: tea-svc + port: 80 + subroutes: # subroute policies + - path: /tea + policies: + - name: policy4 + namespace: tea + action: + pass: tea + ``` + + For VirtualServerRoute, you can apply a policy to a subroute (subroute policies). + + Subroute policies of the same type override spec policies. In the example above, if the type of the policies `policy-1` (in the VirtualServer) and `policy-4` is `accessControl`, then for requests to `cafe.example.com/tea`, NGINX will apply `policy-4`. As with the VirtualServer, the overriding is enforced by NGINX. + + Subroute policies always override route policies no matter the types. For example, the policy `policy-2` in the VirtualServer route will be ignored for the subroute `/tea`, because the subroute has its own policies (in our case, only one policy `policy4`). If the subroute didn't have any policies, then the `policy-2` would be applied. This overriding is enforced by NGINX Ingress Controller -- the `location` context for the subroute will either have route policies or subroute policies, but not both. + +### Invalid Policies + +NGINX will treat a policy as invalid if one of the following conditions is met: + +- The policy doesn't pass the [comprehensive validation](#comprehensive-validation). +- The policy isn't present in the cluster. +- The policy doesn't meet its type-specific requirements. For example, an `ingressMTLS` policy requires TLS termination enabled in the VirtualServer. + +For an invalid policy, NGINX returns the 500 status code for client requests with the following rules: + +- If a policy is referenced in a VirtualServer `route` or a VirtualServerRoute `subroute`, then NGINX will return the 500 status code for requests for the URIs of that route/subroute. +- If a policy is referenced in the VirtualServer `spec`, then NGINX will return the 500 status code for requests for all URIs of that VirtualServer. + +If a policy is invalid, the VirtualServer or VirtualServerRoute will have the [status](/nginx-ingress-controller/configuration/global-configuration/reporting-resources-status#virtualserver-and-virtualserverroute-resources) with the state `Warning` and the message explaining why the policy wasn't considered invalid. + +### Validation + +Two types of validation are available for the Policy resource: + +- *Structural validation*, done by `kubectl` and the Kubernetes API server. +- *Comprehensive validation*, done by NGINX Ingress Controller. + +#### Structural Validation + +The custom resource definition for the Policy includes a structural OpenAPI schema, which describes the type of every field of the resource. + +If you try to create (or update) a resource that violates the structural schema -- for example, the resource uses a string value instead of an array of strings in the `allow` field -- `kubectl` and the Kubernetes API server will reject the resource. + +- Example of `kubectl` validation: + + ```shell + kubectl apply -f access-control-policy-allow.yaml + + error: error validating "access-control-policy-allow.yaml": error validating data: ValidationError(Policy.spec.accessControl.allow): invalid type for org.nginx.k8s.v1.Policy.spec.accessControl.allow: got "string", expected "array"; if you choose to ignore these errors, turn validation off with --validate=false + ``` + +- Example of Kubernetes API server validation: + + ```shell + kubectl apply -f access-control-policy-allow.yaml --validate=false + + The Policy "webapp-policy" is invalid: spec.accessControl.allow: Invalid value: "string": spec.accessControl.allow in body must be of type array: "string" + ``` + +If a resource passes structural validation, then NGINX Ingress Controller's comprehensive validation runs. + +#### Comprehensive Validation + +NGINX Ingress Controller validates the fields of a Policy resource. If a resource is invalid, NGINX Ingress Controller will reject it. The resource will continue to exist in the cluster, but NGINX Ingress Controller will ignore it. + +You can use `kubectl` to check whether or not NGINX Ingress Controller successfully applied a Policy configuration. For our example `webapp-policy` Policy, we can run: + +```shell +kubectl describe pol webapp-policy + +. . . +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 11s nginx-ingress-controller Policy default/webapp-policy was added or updated +``` + +Note how the events section includes a Normal event with the AddedOrUpdated reason that informs us that the configuration was successfully applied. + +If you create an invalid resource, NGINX Ingress Controller will reject it and emit a Rejected event. For example, if you create a Policy `webapp-policy` with an invalid IP `10.0.0.` in the `allow` field, you will get: + +```shell +kubectl describe policy webapp-policy + +. . . +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 7s nginx-ingress-controller Policy default/webapp-policy is invalid and was rejected: spec.accessControl.allow[0]: Invalid value: "10.0.0.": must be a CIDR or IP +``` + +Note how the events section includes a Warning event with the Rejected reason. + +Additionally, this information is also available in the `status` field of the Policy resource. Note the Status section of the Policy: + +```shell +kubectl describe pol webapp-policy + +. . . +Status: + Message: Policy default/webapp-policy is invalid and was rejected: spec.accessControl.allow[0]: Invalid value: "10.0.0.": must be a CIDR or IP + Reason: Rejected + State: Invalid +``` + +**Note**: If you make an existing resource invalid, NGINX Ingress Controller will reject it. diff --git a/content/includes/nic/configuration/security.md b/content/includes/nic/configuration/security.md new file mode 100644 index 000000000..ae2f9648f --- /dev/null +++ b/content/includes/nic/configuration/security.md @@ -0,0 +1,102 @@ +--- +docs: DOCS-597 +doctypes: +- '' +title: Security recommendations +toc: true +weight: 300 +--- + +F5 NGINX Ingress Controller follows Kubernetes best practices: this page outlines configuration specific to NGINX Ingress Controller you may require, including links to examples in the [GitHub repository](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples). + +For general guidance, we recommend the official Kubernetes documentation for [Securing a Cluster](https://kubernetes.io/docs/tasks/administer-cluster/securing-a-cluster/). + +## Kubernetes recommendations + +### RBAC and Service Accounts + +Kubernetes uses [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) to control the resources and operations available to different types of users. + +NGINX Ingress Controller requires RBAC to configure a [ServiceUser](https://kubernetes.io/docs/concepts/security/service-accounts/#default-service-accounts), and provides least privilege access in its standard deployment configurations: + +- [Helm](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/deployments/rbac/rbac.yaml) +- [Manifests](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/deployments/rbac/rbac.yaml) + +By default, the ServiceAccount has access to all Secret resources in the cluster. + +### Secrets + +[Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) are required by NGINX Ingress Controller for certificates and privacy keys, which Kubernetes stores unencrypted by default. We recommend following the [Kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) to store these Secrets using at-rest encryption. + + +## NGINX Ingress Controller recommendations + +### Configure root filesystem as read-only + +{{< caution >}} + This feature is compatible with [NGINX App Protect WAFv5](https://docs.nginx.com/nginx-app-protect-waf/v5/). It is not compatible with [NGINX App Protect WAFv4](https://docs.nginx.com/nginx-app-protect-waf/v4/) or [NGINX App Protect DoS](https://docs.nginx.com/nginx-app-protect-dos/). +{{< /caution >}} + +NGINX Ingress Controller is designed to be resilient against attacks in various ways, such as running the service as non-root to avoid changes to files. We recommend setting filesystems on all containers to read-only, this includes `nginx-ingress-controller`, though also includes `waf-enforcer` and `waf-config-mgr` when NGINX App Protect WAFv5 is in use. This is so that the attack surface is further reduced by limiting changes to binaries and libraries. + +This is not enabled by default, but can be enabled with **Helm** using the [**readOnlyRootFilesystem**]({{< relref "installation/installing-nic/installation-with-helm.md#configuration" >}}) argument in security contexts on all containers: `nginx-ingress-controller`, `waf_enforcer` and `waf_config_mgr`. + +For **Manifests**, uncomment the following sections of the deployment and add sections for `waf-enforcer` and `waf-config-mgr` containers: + +- `readOnlyRootFilesystem: true` +- The entire **volumeMounts** section +- The entire **initContainers** section + +The block below shows the code you will look for: + +```yaml +# volumes: +# - name: nginx-etc +# emptyDir: {} +# - name: nginx-cache +# emptyDir: {} +# - name: nginx-lib +# emptyDir: {} +# - name: nginx-log +# emptyDir: {} +. +. +. +# readOnlyRootFilesystem: true +. +. +. +# volumeMounts: +# - mountPath: /etc/nginx +# name: nginx-etc +# - mountPath: /var/cache/nginx +# name: nginx-cache +# - mountPath: /var/lib/nginx +# name: nginx-lib +# - mountPath: /var/log/nginx +# name: nginx-log +``` + +- Add **waf-enforcer** and **waf-config-mgr** container sections +- Add `readOnlyFilesystem: true` in both containers security context sections + +### Prometheus + +If Prometheus metrics are [enabled]({{< relref "/logging-and-monitoring/prometheus.md" >}}), we recommend [using HTTPS]({{< ref "/nic/configuration/global-configuration/command-line-arguments.md#cmdoption-prometheus-tls-secret" >}}). + +### Snippets + +Snippets allow raw NGINX configuration to be inserted into resources. They are intended for advanced NGINX users and could create vulnerabilities in a cluster if misused. + +Snippets are disabled by default. To use snippets, set the [**enable-snippets**]({{< relref"configuration/global-configuration/command-line-arguments.md#cmdoption-enable-snippets" >}}) command-line argument. + +{{< caution >}} + Snippets are **always** enabled for ConfigMap. +{{< /caution >}} + +For more information, read the following: + +- [Advanced configuration using Snippets]({{< relref "/configuration/ingress-resources/advanced-configuration-with-snippets.md" >}}) +- [Using Snippets with VirtualServer/VirtualServerRoute]({{< ref "/nic/configuration/virtualserver-and-virtualserverroute-resources.md#using-snippets" >}}) +- [Using Snippets with TransportServer]({{< relref "/configuration/transportserver-resource.md#using-snippets" >}}) +- [ConfigMap snippets and custom templates]({{< ref "/nic/configuration/global-configuration/configmap-resource.md#snippets-and-custom-templates" >}}) diff --git a/content/includes/nic/configuration/transportserver-resource.md b/content/includes/nic/configuration/transportserver-resource.md new file mode 100644 index 000000000..b725ea2e6 --- /dev/null +++ b/content/includes/nic/configuration/transportserver-resource.md @@ -0,0 +1,416 @@ +--- +title: TransportServer resources +toc: true +weight: 600 +docs: DOCS-598 +--- + +This document is reference material for the TransportServer resource used by F5 NGINX Ingress Controller. + +The TransportServer resource allows you to configure TCP, UDP, and TLS Passthrough load balancing. The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + +The GitHub repository has [examples of the resources](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources) for specific use cases. + +## Prerequisites + +- For TCP and UDP, the TransportServer resource must be used in conjunction with the [GlobalConfiguration resource]({{< ref "/nic/configuration/global-configuration/globalconfiguration-resource.md" >}}), which must be created separately. +- For TLS Passthrough, make sure to enable the [`-enable-tls-passthrough`]({{< ref "/nic/configuration/global-configuration/command-line-arguments#cmdoption-enable-tls-passthrough.md" >}}) command-line argument of NGINX Ingress Controller. + +## TransportServer Specification + +The TransportServer resource defines load balancing configuration for TCP, UDP, or TLS Passthrough traffic. Below are a few examples: + +- TCP load balancing: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: dns-tcp + spec: + host: dns.example.com + listener: + name: dns-tcp + protocol: TCP + tls: + secret: cafe-secret + upstreams: + - name: dns-app + service: dns-service + port: 5353 + action: + pass: dns-app + ``` + +- UDP load balancing: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: dns-udp + spec: + listener: + name: dns-udp + protocol: UDP + upstreams: + - name: dns-app + service: dns-service + port: 5353 + upstreamParameters: + udpRequests: 1 + udpResponses: 1 + action: + pass: dns-app + ``` + +- TLS passthrough load balancing: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: secure-app + spec: + listener: + name: tls-passthrough + protocol: TLS_PASSTHROUGH + host: app.example.com + upstreams: + - name: secure-app + service: secure-app + port: 8443 + action: + pass: secure-app + ``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``listener`` | The listener on NGINX that will accept incoming connections/datagrams. | [listener](#listener) | Yes | +|``host`` | The host (domain name) of the server. Must be a valid subdomain as defined in RFC 1123, such as ``my-app`` or ``hello.example.com``. Wildcard domains like ``*.example.com`` are not allowed. When specified, NGINX will use this host for SNI-based routing. For TLS Passthrough, this field is required. For TCP with TLS termination, specifying the host enables SNI routing and requires specifying a TLS secret.| ``string`` | No | +|``tls`` | The TLS termination configuration. Not supported for TLS Passthrough load balancing. | [tls](#tls) | No | +|``upstreams`` | A list of upstreams. | [[]upstream](#upstream) | Yes | +|``upstreamParameters`` | The upstream parameters. | [upstreamParameters](#upstreamparameters) | No | +|``action`` | The action to perform for a client connection/datagram. | [action](#action) | Yes | +|``ingressClassName`` | Specifies which Ingress Controller must handle the TransportServer resource. | ``string`` | No | +|``streamSnippets`` | Sets a custom snippet in the ``stream`` context. | ``string`` | No | +|``serverSnippets`` | Sets a custom snippet in the ``server`` context. | ``string`` | No | +{{}} + +\* -- Required for TLS Passthrough load balancing. + +### Listener + +The listener field references a listener that NGINX will use to accept incoming traffic for the TransportServer. For TCP and UDP, the listener must be defined in the [GlobalConfiguration resource]({{< ref "/nic/configuration/global-configuration/globalconfiguration-resource.md" >}}). When referencing a listener, both the name and the protocol must match. For TLS Passthrough, use the built-in listener with the name `tls-passthrough` and the protocol `TLS_PASSTHROUGH`. + +The combination of ``spec.listener.name`` and ``spec.host`` must be unique among all TransportServers. If two TransportServers specify the same combination of ``spec.listener.name`` and ``spec.host``, one of them will be rejected to prevent conflicts. In the case where no host is specified, it is considered an empty string. + +An example: + +```yaml +listener: + name: dns-udp + protocol: UDP +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the listener. | ``string`` | Yes | +|``protocol`` | The protocol of the listener. | ``string`` | Yes | +{{}} + +### TLS + +The tls field defines TLS configuration for a TransportServer. When using TLS termination (not TLS Passthrough), you can specify the host field to enable SNI-based routing, allowing multiple applications to share the same listener port and be distinguished by the TLS SNI hostname. Each application can use its own TLS certificate and key specified via the secret field. + +```yaml +secret: cafe-secret +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``secret`` | The name of a secret with a TLS certificate and key. The secret must belong to the same namespace as the TransportServer. The secret must be of the type ``kubernetes.io/tls`` and contain keys named ``tls.crt`` and ``tls.key`` that contain the certificate and private key as described [here](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls). | ``string`` | No | +{{}} + +### Upstream + +The upstream defines a destination for the TransportServer. For example: + +```yaml +name: secure-app +service: secure-app +port: 8443 +maxFails: 3 +maxConns: 100 +failTimeout: 30s +loadBalancingMethod: least_conn +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the upstream. Must be a valid DNS label as defined in RFC 1035. For example, ``hello`` and ``upstream-123`` are valid. The name must be unique among all upstreams of the resource. | ``string`` | Yes | +|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and close client connections/ignore datagrams. | ``string`` | Yes | +|``port`` | The port of the service. If the service doesn't define that port, NGINX will assume the service has zero endpoints and close client connections/ignore datagrams. The port must fall into the range ``1..65535``. | ``int`` | Yes | +|``maxFails`` | Sets the [number](https://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#max_fails) of unsuccessful attempts to communicate with the server that should happen in the duration set by the failTimeout parameter to consider the server unavailable. The default ``1``. | ``int`` | No | +|``maxConns`` | Sets the [number](https://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#max_conns) of maximum connections to the proxied server. Default value is zero, meaning there is no limit. The default is ``0``. | ``int`` | No | +|``failTimeout`` | Sets the [time](https://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#fail_timeout) during which the specified number of unsuccessful attempts to communicate with the server should happen to consider the server unavailable and the period of time the server will be considered unavailable. The default is ``10s``. | ``string`` | No | +|``healthCheck`` | The health check configuration for the Upstream. See the [health_check](https://nginx.org/en/docs/stream/ngx_stream_upstream_hc_module.html#health_check) directive. Note: this feature is supported only in NGINX Plus. | [healthcheck](#upstreamhealthcheck) | No | +|``loadBalancingMethod`` | The method used to load balance the upstream servers. By default, connections are distributed between the servers using a weighted round-robin balancing method. See the [upstream](http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#upstream) section for available methods and their details. | ``string`` | No | +|``backup`` | The name of the backup service of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). This will be used when the primary servers are unavailable. Note: The parameter cannot be used along with the ``random`` , ``hash`` or ``ip_hash`` load balancing methods. | ``string`` | No | +|``backupPort`` | The port of the backup service. The backup port is required if the backup service name is provided. The port must fall into the range ``1..65535``. | ``uint16`` | No | +{{}} + +### Upstream.Healthcheck + +The Healthcheck defines an [active health check](https://nginx.org/en/docs/stream/ngx_stream_upstream_hc_module.html?#health_check). In the example below we enable a health check for an upstream and configure all the available parameters: + +```yaml +name: secure-app +service: secure-app +port: 8443 +healthCheck: + enable: true + interval: 20s + timeout: 30s + jitter: 3s + fails: 5 + passes: 5 + port: 8080 +``` + +{{< note >}} This feature is only supported with NGINX Plus. {{< /note >}} + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables a health check for an upstream server. The default is ``false``. | ``boolean`` | No | +|``interval`` | The interval between two consecutive health checks. The default is ``5s``. | ``string`` | No | +|``timeout`` | This overrides the timeout set by [proxy_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout) which is set in `SessionParameters` for health checks. The default value is ``5s``. | ``string`` | No | +|``jitter`` | The time within which each health check will be randomly delayed. By default, there is no delay. | ``string`` | No | +|``fails`` | The number of consecutive failed health checks of a particular upstream server after which this server will be considered unhealthy. The default is ``1``. | ``integer`` | No | +|``passes`` | The number of consecutive passed health checks of a particular upstream server after which the server will be considered healthy. The default is ``1``. | ``integer`` | No | +|``port`` | The port used for health check requests. By default, the [server port is used](https://nginx.org/en/docs/stream/ngx_stream_upstream_hc_module.html#health_check_port). Note: in contrast with the port of the upstream, this port is not a service port, but a port of a pod. | ``integer`` | No | +|``match`` | Controls the data to send and the response to expect for the healthcheck. | [match](#upstreamhealthcheckmatch) | No | +{{}} + +### Upstream.Healthcheck.Match + +The match controls the data to send and the response to expect for the healthcheck: + +```yaml +match: + send: 'GET / HTTP/1.0\r\nHost: localhost\r\n\r\n' + expect: "~200 OK" +``` + +Both `send` and `expect` fields can contain hexadecimal literals with the prefix `\x` followed by two hex digits, for example, `\x80`. + +See the [match](https://nginx.org/en/docs/stream/ngx_stream_upstream_hc_module.html#match) directive for details. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``send`` | A string to send to an upstream server. | ``string`` | No | +|``expect`` | A literal string or a regular expression that the data obtained from the server should match. The regular expression is specified with the preceding ``~*`` modifier (for case-insensitive matching), or the ``~`` modifier (for case-sensitive matching). NGINX Ingress Controller validates a regular expression using the RE2 syntax. | ``string`` | No | +{{}} + +### UpstreamParameters + +The upstream parameters define various parameters for the upstreams: + +```yaml +upstreamParameters: + udpRequests: 1 + udpResponses: 1 + connectTimeout: 60s + nextUpstream: true + nextUpstreamTimeout: 50s + nextUpstreamTries: 1 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``udpRequests`` | The number of datagrams, after receiving which, the next datagram from the same client starts a new session. See the [proxy_requests](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_requests) directive. The default is ``0``. | ``int`` | No | +|``udpResponses`` | The number of datagrams expected from the proxied server in response to a client datagram. See the [proxy_responses](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_responses) directive. By default, the number of datagrams is not limited. | ``int`` | No | +|``connectTimeout`` | The timeout for establishing a connection with a proxied server. See the [proxy_connect_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_connect_timeout) directive. The default is ``60s``. | ``string`` | No | +|``nextUpstream`` | If a connection to the proxied server cannot be established, determines whether a client connection will be passed to the next server. See the [proxy_next_upstream](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream) directive. The default is ``true``. | bool | No | +|``nextUpstreamTries`` | The number of tries for passing a connection to the next server. See the [proxy_next_upstream_tries](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_tries) directive. The default is ``0``. | ``int`` | No | +|``nextUpstreamTimeout`` | The time allowed to pass a connection to the next server. See the [proxy_next_upstream_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_timeout) directive. The default us ``0``. | ``string`` | No | +{{}} + +### SessionParameters + +The session parameters define various parameters for TCP connections and UDP sessions. + +```yaml +sessionParameters: + timeout: 50s +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``timeout`` | The timeout between two successive read or write operations on client or proxied server connections. See [proxy_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout) directive. The default is ``10m``. | ``string`` | No | +{{}} + +### Action + +The action defines an action to perform for a client connection/datagram. + +In the example below, client connections/datagrams are passed to an upstream `dns-app`: + +```yaml +action: + pass: dns-app +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``pass`` | Passes connections/datagrams to an upstream. The upstream with that name must be defined in the resource. | ``string`` | Yes | +{{}} + +## Using TransportServer + +You can use the usual `kubectl` commands to work with TransportServer resources, similar to Ingress resources. + +For example, the following command creates a TransportServer resource defined in `transport-server-passthrough.yaml` with the name `secure-app`: + +```shell +kubectl apply -f transport-server-passthrough.yaml +``` +```text +transportserver.k8s.nginx.org/secure-app created +``` + +You can get the resource by running: + +```shell +kubectl get transportserver secure-app +``` +```text +NAME AGE +secure-app 46sm +``` + +In the kubectl get and similar commands, you can also use the short name `ts` instead of `transportserver`. + +### Using Snippets + +Snippets allow you to insert raw NGINX config into different contexts of NGINX configuration. In the example below, we use snippets to configure [access control](http://nginx.org/en/docs/stream/ngx_stream_access_module.html) in a TransportServer: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: TransportServer +metadata: + name: cafe +spec: + host: cafe.example.com + serverSnippets: | + deny 192.168.1.1; + allow 192.168.1.0/24; + upstreams: + - name: tea + service: tea-svc + port: 80 +``` + +Snippets can also be specified for a stream. In the example below, we use snippets to [limit the number of connections](https://nginx.org/en/docs/stream/ngx_stream_limit_conn_module.html): + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: TransportServer +metadata: + name: cafe +spec: + host: cafe.example.com + streamSnippets: limit_conn_zone $binary_remote_addr zone=addr:10m; + serverSnippets: limit_conn addr 1; + upstreams: + - name: tea + service: tea-svc + port: 80 +``` + +{{< note >}} To configure snippets in the `stream` context, use `stream-snippets` ConfigMap key. {{< /note >}} + +For additional information, view the [Advanced configuration with Snippets]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md" >}}) topic. + +### Validation + +Two types of validation are available for the TransportServer resource: + +- *Structural validation* by the `kubectl` and Kubernetes API server. +- *Comprehensive validation* by NGINX Ingress Controller. + +#### Structural Validation + +The custom resource definition for the TransportServer includes structural OpenAPI schema which describes the type of every field of the resource. + +If you try to create (or update) a resource that violates the structural schema (for example, you use a string value for the port field of an upstream), `kubectl` and Kubernetes API server will reject such a resource: + +- Example of `kubectl` validation: + + ```shell + kubectl apply -f transport-server-passthrough.yaml + ``` + ```text + error: error validating "transport-server-passthrough.yaml": error validating data: ValidationError(TransportServer.spec.upstreams[0].port): invalid type for org.nginx.k8s.v1.TransportServer.spec.upstreams.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + ``` + +- Example of Kubernetes API server validation: + + ```shell + kubectl apply -f transport-server-passthrough.yaml --validate=false + ``` + ```text + The TransportServer "secure-app" is invalid: []: Invalid value: map[string]interface {}{ ... }: validation failure list: + spec.upstreams.port in body must be of type integer: "string" + ``` + +If a resource is not rejected (it doesn't violate the structural schema), NGINX Ingress Controller will validate it further. + +#### Comprehensive Validation + +NGINX Ingress Controller validates the fields of a TransportServer resource. If a resource is invalid, NGINX Ingress Controller will reject it: the resource will continue to exist in the cluster, but NGINX Ingress Controller will ignore it. + +You can check if NGINX Ingress Controller successfully applied the configuration for a TransportServer. For our example `secure-app` TransportServer, we can run: + +```shell +kubectl describe ts secure-app +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 3s nginx-ingress-controller Configuration for default/secure-app was added or updated +``` + +Note how the events section includes a Normal event with the AddedOrUpdated reason that informs us that the configuration was successfully applied. + +If you create an invalid resource, NGINX Ingress Controller will reject it and emit a Rejected event. For example, if you create a TransportServer `secure-app` with a pass action that references a non-existing upstream, you will get : + +```shell +kubectl describe ts secure-app +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 2s nginx-ingress-controller TransportServer default/secure-app is invalid and was rejected: spec.action.pass: Not found: "some-app" +``` + +Note how the events section includes a Warning event with the Rejected reason. + +**Note**: If you make an existing resource invalid, NGINX Ingress Controller will reject it and remove the corresponding configuration from NGINX. + +## Customization via ConfigMap + +The [ConfigMap]({{< ref "/nic/configuration/global-configuration/configmap-resource.md" >}}) keys (except for `stream-snippets`, `stream-log-format`, `resolver-addresses`, `resolver-ipv6`, `resolver-valid` and `resolver-timeout`) do not affect TransportServer resources. diff --git a/content/includes/nic/configuration/virtualserver-and-virtualserverroute-resources.md b/content/includes/nic/configuration/virtualserver-and-virtualserverroute-resources.md new file mode 100644 index 000000000..1acd05b02 --- /dev/null +++ b/content/includes/nic/configuration/virtualserver-and-virtualserverroute-resources.md @@ -0,0 +1,1101 @@ +--- +title: VirtualServer and VirtualServerRoute resources +toc: true +weight: 700 +docs: DOCS-599 +--- + +This document is reference material for the VirtualServer and VirtualServerRoute resources used by F5 NGINX Ingress Controller. + +VirtualServer and VirtualServerRoute resources are load balancing configurations recommended as an alternative to the Ingress resource. + +They enable use cases not supported with the Ingress resource, such as traffic splitting and advanced content-based routing. The resources are implemented as [Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + +The GitHub repository has [examples of the resources](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources) for specific use cases. + +--- + +## VirtualServer specification + +The VirtualServer resource defines load balancing configuration for a domain name, such as `example.com`. Below is an example of such configuration: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: cafe +spec: + host: cafe.example.com + listener: + http: http-8083 + https: https-8443 + tls: + secret: cafe-secret + gunzip: on + upstreams: + - name: tea + service: tea-svc + port: 80 + - name: coffee + service: coffee-svc + port: 80 + routes: + - path: /tea + action: + pass: tea + - path: /coffee + action: + pass: coffee + - path: ~ ^/decaf/.*\\.jpg$ + action: + pass: coffee + - path: = /green/tea + action: + pass: tea +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``host`` | The host (domain name) of the server. Must be a valid subdomain as defined in RFC 1123, such as ``my-app`` or ``hello.example.com``. When using a wildcard domain like ``*.example.com`` the domain must be contained in double quotes. The ``host`` value needs to be unique among all Ingress and VirtualServer resources. See also [Handling Host and Listener Collisions](/nginx-ingress-controller/configuration/host-and-listener-collisions). | ``string`` | Yes | +|``listener`` | Sets a custom HTTP and/or HTTPS listener. Valid fields are `listener.http` and `listener.https`. Each field must reference the name of a valid listener defined in a GlobalConfiguration resource | [listener](#virtualserverlistener) | No | +|``tls`` | The TLS termination configuration. | [tls](#virtualservertls) | No | +|``gunzip`` | Enables or disables [decompression](https://docs.nginx.com/nginx/admin-guide/web-server/compression/) of gzipped responses for clients. Allowed values “on”/“off”, “true”/“false” or “yes”/“no”. If the ``gunzip`` value is not set, it defaults to ``off``. | ``boolean`` | No | +|``externalDNS`` | The externalDNS configuration for a VirtualServer. | [externalDNS](#virtualserverexternaldns) | No | +|``dos`` | A reference to a DosProtectedResource, setting this enables DOS protection of the VirtualServer. | ``string`` | No | +|``policies`` | A list of policies. | [[]policy](#virtualserverpolicy) | No | +|``upstreams`` | A list of upstreams. | [[]upstream](#upstream) | No | +|``routes`` | A list of routes. | [[]route](#virtualserverroute) | No | +|``ingressClassName`` | Specifies which Ingress Controller must handle the VirtualServer resource. | ``string`` | No | +|``internalRoute`` | Specifies if the VirtualServer resource is an internal route or not. | ``boolean`` | No | +|``http-snippets`` | Sets a custom snippet in the http context. | ``string`` | No | +|``server-snippets`` | Sets a custom snippet in server context. Overrides the ``server-snippets`` ConfigMap key. | ``string`` | No | +{{}} + +### VirtualServer.TLS + +The tls field defines TLS configuration for a VirtualServer. For example: + +```yaml +secret: cafe-secret +redirect: + enable: true +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``secret`` | The name of a secret with a TLS certificate and key. The secret must belong to the same namespace as the VirtualServer. The secret must be of the type ``kubernetes.io/tls`` and contain keys named ``tls.crt`` and ``tls.key`` that contain the certificate and private key as described [here](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls). If the secret doesn't exist or is invalid, NGINX will break any attempt to establish a TLS connection to the host of the VirtualServer. If the secret is not specified but [wildcard TLS secret](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments#cmdoption-wildcard-tls-secret) is configured, NGINX will use the wildcard secret for TLS termination. | ``string`` | No | +|``redirect`` | The redirect configuration of the TLS for a VirtualServer. | [tls.redirect](#virtualservertlsredirect) | No | ### VirtualServer.TLS.Redirect | +|``cert-manager`` | The cert-manager configuration of the TLS for a VirtualServer. | [tls.cert-manager](#virtualservertlscertmanager) | No | ### VirtualServer.TLS.CertManager | +{{}} + +### VirtualServer.TLS.Redirect + +The redirect field configures a TLS redirect for a VirtualServer: + +```yaml +enable: true +code: 301 +basedOn: scheme +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables a TLS redirect for a VirtualServer. The default is ``False``. | ``boolean`` | No | +|``code`` | The status code of a redirect. The allowed values are: ``301`` , ``302`` , ``307`` , ``308``. The default is ``301``. | ``int`` | No | +|``basedOn`` | The attribute of a request that NGINX will evaluate to send a redirect. The allowed values are ``scheme`` (the scheme of the request) or ``x-forwarded-proto`` (the ``X-Forwarded-Proto`` header of the request). The default is ``scheme``. | ``string`` | No | ### VirtualServer.Policy | +{{}} + +### VirtualServer.TLS.CertManager + +The cert-manager field configures x509 automated Certificate management for VirtualServer resources using cert-manager (cert-manager.io). Please see the [cert-manager configuration documentation](https://cert-manager.io/docs/configuration/) for more information on deploying and configuring Issuers. Example: + +```yaml +cert-manager: + cluster-issuer: "my-issuer-name" +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``issuer`` | the name of an Issuer. An Issuer is a cert-manager resource which describes the certificate authority capable of signing certificates. The Issuer must be in the same namespace as the VirtualServer resource. Please note that one of `issuer` and `cluster-issuer` are required, but they are mutually exclusive - one and only one must be defined. | ``string`` | No | +|``cluster-issuer`` | the name of a ClusterIssuer. A ClusterIssuer is a cert-manager resource which describes the certificate authority capable of signing certificates. It does not matter which namespace your VirtualServer resides, as ClusterIssuers are non-namespaced resources. Please note that one of `issuer` and `cluster-issuer` are required, but they are mutually exclusive - one and only one must be defined. | ``string`` | No | +|``issuer-kind`` | The kind of the external issuer resource, for example AWSPCAIssuer. This is only necessary for out-of-tree issuers. This cannot be defined if `cluster-issuer` is also defined. | ``string`` | No | +|``issuer-group`` | The API group of the external issuer controller, for example awspca.cert-manager.io. This is only necessary for out-of-tree issuers. This cannot be defined if `cluster-issuer` is also defined. | ``string`` | No | +|``common-name`` | This field allows you to configure spec.commonName for the Certificate to be generated. This configuration adds a CN to the x509 certificate. | ``string`` | No | +|``duration`` | This field allows you to configure spec.duration field for the Certificate to be generated. Must be specified using a [Go time.Duration](https://pkg.go.dev/time#ParseDuration) string format, which does not allow the d (days) suffix. You must specify these values using s, m, and h suffixes instead. | ``string`` | No | +|``renew-before`` | this annotation allows you to configure spec.renewBefore field for the Certificate to be generated. Must be specified using a [Go time.Duration](https://pkg.go.dev/time#ParseDuration) string format, which does not allow the d (days) suffix. You must specify these values using s, m, and h suffixes instead. | ``string`` | No | +|``usages`` | This field allows you to configure spec.usages field for the Certificate to be generated. Pass a string with comma-separated values i.e. ``key agreement,digital signature, server auth``. An exhaustive list of supported key usages can be found in the [the cert-manager api documentation](https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.KeyUsage). | ``string`` | No | +|``issue-temp-cert`` | When ``true``, ask cert-manager for a [temporary self-signed certificate](https://cert-manager.io/docs/usage/certificate/#temporary-certificates-while-issuing) pending the issuance of the Certificate. This allows HTTPS-only servers to use ACME HTTP01 challenges when the TLS secret does not exist yet. | ``boolean`` | No | +{{}} + +### VirtualServer.Listener +The listener field defines a custom HTTP and/or HTTPS listener. +The respective listeners used must reference the name of a listener defined using a [GlobalConfiguration](/nginx-ingress-controller/configuration/global-configuration/globalconfiguration-resource/) resource. +For example: +```yaml +http: http-8083 +https: https-8443 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``http`` | The name of am HTTP listener defined in a [GlobalConfiguration](/nginx-ingress-controller/configuration/global-configuration/globalconfiguration-resource/) resource. | ``string`` | No | +|``https`` | The name of an HTTPS listener defined in a [GlobalConfiguration](/nginx-ingress-controller/configuration/global-configuration/globalconfiguration-resource/) resource. | ``string`` | No | +{{}} + +### VirtualServer.ExternalDNS + +The externalDNS field configures controlling DNS records dynamically for VirtualServer resources using [ExternalDNS](https://github.com/kubernetes-sigs/external-dns). Please see the [ExternalDNS configuration documentation](https://kubernetes-sigs.github.io/external-dns/) for more information on deploying and configuring ExternalDNS and Providers. Example: + +```yaml +enable: true +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables ExternalDNS integration for a VirtualServer resource. The default is ``false``. | ``string`` | No | +|``labels`` | Configure labels to be applied to the Endpoint resources that will be consumed by ExternalDNS. | ``map[string]string`` | No | +|``providerSpecific`` | Configure provider specific properties which holds the name and value of a configuration which is specific to individual DNS providers. | [[]ProviderSpecific](#virtualserverexternaldnsproviderspecific) | No | +|``recordTTL`` | TTL for the DNS record. This defaults to 0 if not defined. See [the ExternalDNS TTL documentation for provider-specific defaults](https://kubernetes-sigs.github.io/external-dns/v0.14.2/ttl/#providers) | ``int64`` | No | +|``recordType`` | The record Type that should be created, e.g. "A", "AAAA", "CNAME". This is automatically computed based on the external endpoints if not defined. | ``string`` | No | +{{}} + +### VirtualServer.ExternalDNS.ProviderSpecific + +The providerSpecific field of the externalDNS block allows the specification of provider specific properties which is a list of key value pairs of configurations which are specific to individual DNS providers. Example: + +```yaml +- name: my-name + value: my-value +- name: my-name2 + value: my-value2 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the key value pair. | ``string`` | Yes | +|``value`` | The value of the key value pair. | ``string`` | Yes | +{{}} + +### VirtualServer.Policy + +The policy field references a [Policy resource](/nginx-ingress-controller/configuration/policy-resource/) by its name and optional namespace. For example: + +```yaml +name: access-control +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of a policy. If the policy doesn't exist or invalid, NGINX will respond with an error response with the `500` status code. | ``string`` | Yes | +|``namespace`` | The namespace of a policy. If not specified, the namespace of the VirtualServer resource is used. | ``string`` | No | +{{}} + +### VirtualServer.Route + +The route defines rules for matching client requests to actions like passing a request to an upstream. For example: + +```yaml + path: /tea + action: + pass: tea +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``path`` | The path of the route. NGINX will match it against the URI of a request. Possible values are: a prefix ( ``/`` , ``/path`` ), an exact match ( ``=/exact/match`` ), a case insensitive regular expression ( ``~*^/Bar.*\.jpg`` ) or a case sensitive regular expression ( ``~^/foo.*\.jpg`` ). In the case of a prefix (must start with ``/`` ) or an exact match (must start with ``=`` ), the path must not include any whitespace characters, ``{`` , ``}`` or ``;``. In the case of the regex matches, all double quotes ``"`` must be escaped and the match can't end in an unescaped backslash ``\``. The path must be unique among the paths of all routes of the VirtualServer. Check the [location](https://nginx.org/en/docs/http/ngx_http_core_module.html#location) directive for more information. | ``string`` | Yes | +|``policies`` | A list of policies. The policies override the policies of the same type defined in the ``spec`` of the VirtualServer. See [Applying Policies](/nginx-ingress-controller/configuration/policy-resource/#applying-policies) for more details. | [[]policy](#virtualserverpolicy) | No | +|``action`` | The default action to perform for a request. | [action](#action) | No | +|``dos`` | A reference to a DosProtectedResource, setting this enables DOS protection of the VirtualServer route. | ``string`` | No | +|``splits`` | The default splits configuration for traffic splitting. Must include at least 2 splits. | [[]split](#split) | No | +|``matches`` | The matching rules for advanced content-based routing. Requires the default ``action`` or ``splits``. Unmatched requests will be handled by the default ``action`` or ``splits``. | [matches](#match) | No | +|``route`` | The name of a VirtualServerRoute resource that defines this route. If the VirtualServerRoute belongs to a different namespace than the VirtualServer, you need to include the namespace. For example, ``tea-namespace/tea``. | ``string`` | No | +|``errorPages`` | The custom responses for error codes. NGINX will use those responses instead of returning the error responses from the upstream servers or the default responses generated by NGINX. A custom response can be a redirect or a canned response. For example, a redirect to another URL if an upstream server responded with a 404 status code. | [[]errorPage](#errorpage) | No | +|``location-snippets`` | Sets a custom snippet in the location context. Overrides the ``location-snippets`` ConfigMap key. | ``string`` | No | +{{}} + +\* -- a route must include exactly one of the following: `action`, `splits`, or `route`. + +## VirtualServerRoute specification + +The VirtualServerRoute resource defines a route for a VirtualServer. It can consist of one or multiple subroutes. The VirtualServerRoute is an alternative to [Mergeable Ingress types](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration). + +In the example below, the VirtualServer `cafe` from the namespace `cafe-ns` defines a route with the path `/coffee`, which is further defined in the VirtualServerRoute `coffee` from the namespace `coffee-ns`. + +VirtualServer: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: cafe + namespace: cafe-ns +spec: + host: cafe.example.com + upstreams: + - name: tea + service: tea-svc + port: 80 + routes: + - path: /tea + action: + pass: tea + - path: /coffee + route: coffee-ns/coffee +``` + +VirtualServerRoute: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServerRoute +metadata: + name: coffee + namespace: coffee-ns +spec: + host: cafe.example.com + upstreams: + - name: latte + service: latte-svc + port: 80 + - name: espresso + service: espresso-svc + port: 80 + subroutes: + - path: /coffee/latte + action: + pass: latte + - path: /coffee/espresso + action: + pass: espresso +``` + +Note that each subroute must have a `path` that starts with the same prefix (here `/coffee`), which is defined in the route of the VirtualServer. Additionally, the `host` in the VirtualServerRoute must be the same as the `host` of the VirtualServer. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``host`` | The host (domain name) of the server. Must be a valid subdomain as defined in RFC 1123, such as ``my-app`` or ``hello.example.com``. When using a wildcard domain like ``*.example.com`` the domain must be contained in double quotes. Must be the same as the ``host`` of the VirtualServer that references this resource. | ``string`` | Yes | +|``upstreams`` | A list of upstreams. | [[]upstream](#upstream) | No | +|``subroutes`` | A list of subroutes. | [[]subroute](#virtualserverroutesubroute) | No | +|``ingressClassName`` | Specifies which Ingress Controller must handle the VirtualServerRoute resource. Must be the same as the ``ingressClassName`` of the VirtualServer that references this resource. | ``string``_ | No | +{{}} + +### VirtualServerRoute.Subroute + +The subroute defines rules for matching client requests to actions like passing a request to an upstream. For example: + +```yaml +path: /coffee +action: + pass: coffee +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``path`` | The path of the subroute. NGINX will match it against the URI of a request. Possible values are: a prefix ( ``/`` , ``/path`` ), an exact match ( ``=/exact/match`` ), a case insensitive regular expression ( ``~*^/Bar.*\.jpg`` ) or a case sensitive regular expression ( ``~^/foo.*\.jpg`` ). In the case of a prefix, the path must start with the same path as the path of the route of the VirtualServer that references this resource. In the case of an exact or regex match, the path must be the same as the path of the route of the VirtualServer that references this resource. A matching path of the route of the VirtualServer but in different type is not accepted, e.g. a regex path (`~/match`) cannot be used with a prefix path in VirtualServer (`/match`) In the case of a prefix or an exact match, the path must not include any whitespace characters, ``{`` , ``}`` or ``;``. In the case of the regex matches, all double quotes ``"`` must be escaped and the match can't end in an unescaped backslash ``\``. The path must be unique among the paths of all subroutes of the VirtualServerRoute. | ``string`` | Yes | +|``policies`` | A list of policies. The policies override *all* policies defined in the route of the VirtualServer that references this resource. The policies also override the policies of the same type defined in the ``spec`` of the VirtualServer. See [Applying Policies](/nginx-ingress-controller/configuration/policy-resource/#applying-policies) for more details. | [[]policy](#virtualserverpolicy) | No | +|``action`` | The default action to perform for a request. | [action](#action) | No | +|``dos`` | A reference to a DosProtectedResource, setting this enables DOS protection of the VirtualServerRoute subroute. | ``string`` | No | +|``splits`` | The default splits configuration for traffic splitting. Must include at least 2 splits. | [[]split](#split) | No | +|``matches`` | The matching rules for advanced content-based routing. Requires the default ``action`` or ``splits``. Unmatched requests will be handled by the default ``action`` or ``splits``. | [matches](#match) | No | +|``errorPages`` | The custom responses for error codes. NGINX will use those responses instead of returning the error responses from the upstream servers or the default responses generated by NGINX. A custom response can be a redirect or a canned response. For example, a redirect to another URL if an upstream server responded with a 404 status code. | [[]errorPage](#errorpage) | No | +|``location-snippets`` | Sets a custom snippet in the location context. Overrides the ``location-snippets`` of the VirtualServer (if set) or the ``location-snippets`` ConfigMap key. | ``string`` | No | +{{}} + +\* -- a subroute must include exactly one of the following: `action` or `splits`. + +## Common VirtualServer and VirtualServerRoute specifications + +### Upstream + +The upstream defines a destination for the routing configuration. For example: + +```yaml +name: tea +service: tea-svc +subselector: + version: canary +port: 80 +lb-method: round_robin +fail-timeout: 10s +max-fails: 1 +max-conns: 32 +keepalive: 32 +connect-timeout: 30s +read-timeout: 30s +send-timeout: 30s +next-upstream: "error timeout non_idempotent" +next-upstream-timeout: 5s +next-upstream-tries: 10 +client-max-body-size: 2m +tls: + enable: true +``` + +**Note**: The WebSocket protocol is supported without any additional configuration. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the upstream. Must be a valid DNS label as defined in RFC 1035. For example, ``hello`` and ``upstream-123`` are valid. The name must be unique among all upstreams of the resource. | ``string`` | Yes | +|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. For NGINX Plus only, services of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) are also supported (check the [prerequisites](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services#prerequisites) ). | ``string`` | Yes | +|``subselector`` | Selects the pods within the service using label keys and values. By default, all pods of the service are selected. Note: the specified labels are expected to be present in the pods when they are created. If the pod labels are updated, NGINX Ingress Controller will not see that change until the number of the pods is changed. | ``map[string]string`` | No | +|``use-cluster-ip`` | Enables using the Cluster IP and port of the service instead of the default behavior of using the IP and port of the pods. When this field is enabled, the fields that configure NGINX behavior related to multiple upstream servers (like ``lb-method`` and ``next-upstream``) will have no effect, as NGINX Ingress Controller will configure NGINX with only one upstream server that will match the service Cluster IP. | ``boolean`` | No | +|``port`` | The port of the service. If the service doesn't define that port, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. The port must fall into the range ``1..65535``. | ``uint16`` | Yes | +|``lb-method`` | The load [balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify ``round_robin``. The default is specified in the ``lb-method`` ConfigMap key. | ``string`` | No | +|``fail-timeout`` | The time during which the specified number of unsuccessful attempts to communicate with an upstream server should happen to consider the server unavailable. See the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the server directive. The default is set in the ``fail-timeout`` ConfigMap key. | ``string`` | No | +|``max-fails`` | The number of unsuccessful attempts to communicate with an upstream server that should happen in the duration set by the ``fail-timeout`` to consider the server unavailable. See the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the server directive. The default is set in the ``max-fails`` ConfigMap key. | ``int`` | No | +|``max-conns`` | The maximum number of simultaneous active connections to an upstream server. See the [max_conns](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_conns) parameter of the server directive. By default there is no limit. Note: if keepalive connections are enabled, the total number of active and idle keepalive connections to an upstream server may exceed the ``max_conns`` value. | ``int`` | No | +|``keepalive`` | Configures the cache for connections to upstream servers. The value ``0`` disables the cache. See the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. The default is set in the ``keepalive`` ConfigMap key. | ``int`` | No | +|``connect-timeout`` | The timeout for establishing a connection with an upstream server. See the [proxy_connect_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) directive. The default is specified in the ``proxy-connect-timeout`` ConfigMap key. | ``string`` | No | +|``read-timeout`` | The timeout for reading a response from an upstream server. See the [proxy_read_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) directive. The default is specified in the ``proxy-read-timeout`` ConfigMap key. | ``string`` | No | +|``send-timeout`` | The timeout for transmitting a request to an upstream server. See the [proxy_send_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout) directive. The default is specified in the ``proxy-send-timeout`` ConfigMap key. | ``string`` | No | +|``next-upstream`` | Specifies in which cases a request should be passed to the next upstream server. See the [proxy_next_upstream](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream) directive. The default is ``error timeout``. | ``string`` | No | +|``next-upstream-timeout`` | The time during which a request can be passed to the next upstream server. See the [proxy_next_upstream_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream_timeout) directive. The ``0`` value turns off the time limit. The default is ``0``. | ``string`` | No | +|``next-upstream-tries`` | The number of possible tries for passing a request to the next upstream server. See the [proxy_next_upstream_tries](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream_tries) directive. The ``0`` value turns off this limit. The default is ``0``. | ``int`` | No | +|``client-max-body-size`` | Sets the maximum allowed size of the client request body. See the [client_max_body_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) directive. The default is set in the ``client-max-body-size`` ConfigMap key. | ``string`` | No | +|``tls`` | The TLS configuration for the Upstream. | [tls](#upstreamtls) | No | +|``healthCheck`` | The health check configuration for the Upstream. See the [health_check](https://nginx.org/en/docs/http/ngx_http_upstream_hc_module.html#health_check) directive. Note: this feature is supported only in NGINX Plus. | [healthcheck](#upstreamhealthcheck) | No | +|``slow-start`` | The slow start allows an upstream server to gradually recover its weight from 0 to its nominal value after it has been recovered or became available or when the server becomes available after a period of time it was considered unavailable. By default, the slow start is disabled. See the [slow_start](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#slow_start) parameter of the server directive. Note: The parameter cannot be used along with the ``random`` , ``hash`` or ``ip_hash`` load balancing methods and will be ignored. | ``string`` | No | +|``queue`` | Configures a queue for an upstream. A client request will be placed into the queue if an upstream server cannot be selected immediately while processing the request. By default, no queue is configured. Note: this feature is supported only in NGINX Plus. | [queue](#upstreamqueue) | No | +|``buffering`` | Enables buffering of responses from the upstream server. See the [proxy_buffering](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) directive. The default is set in the ``proxy-buffering`` ConfigMap key. | ``boolean`` | No | +|``buffers`` | Configures the buffers used for reading a response from the upstream server for a single connection. | [buffers](#upstreambuffers) | No | +|``buffer-size`` | Sets the size of the buffer used for reading the first part of a response received from the upstream server. See the [proxy_buffer_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) directive. The default is set in the ``proxy-buffer-size`` ConfigMap key. | ``string`` | No | +|``ntlm`` | Allows proxying requests with NTLM Authentication. See the [ntlm](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#ntlm) directive. In order for NTLM authentication to work, it is necessary to enable keepalive connections to upstream servers using the ``keepalive`` field. Note: this feature is supported only in NGINX Plus.| ``boolean`` | No | +|``type`` |The type of the upstream. Supported values are ``http`` and ``grpc``. The default is ``http``. For gRPC, it is necessary to enable HTTP/2 in the [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#listeners) and configure TLS termination in the VirtualServer. | ``string`` | No | +|``backup`` | The name of the backup service of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). This will be used when the primary servers are unavailable. Note: The parameter cannot be used along with the ``random`` , ``hash`` or ``ip_hash`` load balancing methods. | ``string`` | No | +|``backupPort`` | The port of the backup service. The backup port is required if the backup service name is provided. The port must fall into the range ``1..65535``. | ``uint16`` | No | +{{}} + +### Upstream.Buffers + +The buffers field configures the buffers used for reading a response from the upstream server for a single connection: + +```yaml +number: 4 +size: 8K +``` + +See the [proxy_buffers](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directive for additional information. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``number`` | Configures the number of buffers. The default is set in the ``proxy-buffers`` ConfigMap key. | ``int`` | Yes | +|``size`` | Configures the size of a buffer. The default is set in the ``proxy-buffers`` ConfigMap key. | ``string`` | Yes | +{{}} + +### Upstream.TLS + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables HTTPS for requests to upstream servers. The default is ``False`` , meaning that HTTP will be used. Note: by default, NGINX will not verify the upstream server certificate. To enable the verification, configure an [EgressMTLS Policy](/nginx-ingress-controller/configuration/policy-resource/#egressmtls). | ``boolean`` | No | +{{}} + +### Upstream.Queue + +The queue field configures a queue. A client request will be placed into the queue if an upstream server cannot be selected immediately while processing the request: + +```yaml +size: 10 +timeout: 60s +``` + +See [`queue`](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#queue) directive for additional information. + +Note: This feature is supported only in NGINX Plus. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``size`` | The size of the queue. | ``int`` | Yes | +|``timeout`` | The timeout of the queue. A request cannot be queued for a period longer than the timeout. The default is ``60s``. | ``string`` | No | +{{}} + +### Upstream.Healthcheck + +The Healthcheck defines an [active health check](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/). In the example below we enable a health check for an upstream and configure all the available parameters, including the `slow-start` parameter combined with [`mandatory` and `persistent`](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#mandatory-health-checks): + +```yaml +name: tea +service: tea-svc +port: 80 +slow-start: 30s +healthCheck: + enable: true + path: /healthz + interval: 20s + jitter: 3s + fails: 5 + passes: 5 + port: 8080 + tls: + enable: true + connect-timeout: 10s + read-timeout: 10s + send-timeout: 10s + headers: + - name: Host + value: my.service + statusMatch: "! 500" + mandatory: true + persistent: true + keepalive-time: 60s +``` + +Note: This feature is supported only in NGINX Plus. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables a health check for an upstream server. The default is ``false``. | ``boolean`` | No | +|``path`` | The path used for health check requests. The default is ``/``. This not configurable for gRPC type upstreams. | ``string`` | No | +|``interval`` | The interval between two consecutive health checks. The default is ``5s``. | ``string`` | No | +|``jitter`` | The time within which each health check will be randomly delayed. By default, there is no delay. | ``string`` | No | +|``fails`` | The number of consecutive failed health checks of a particular upstream server after which this server will be considered unhealthy. The default is ``1``. | ``integer`` | No | +|``passes`` | The number of consecutive passed health checks of a particular upstream server after which the server will be considered healthy. The default is ``1``. | ``integer`` | No | +|``port`` | The port used for health check requests. By default, the [server port is used](https://nginx.org/en/docs/http/ngx_http_upstream_hc_module.html#health_check_port). Note: in contrast with the port of the upstream, this port is not a service port, but a port of a pod. | ``integer`` | No | +|``tls`` | The TLS configuration used for health check requests. By default, the ``tls`` field of the upstream is used. | [upstream.tls](#upstreamtls) | No | +|``connect-timeout`` | The timeout for establishing a connection with an upstream server. By default, the ``connect-timeout`` of the upstream is used. | ``string`` | No | +|``read-timeout`` | The timeout for reading a response from an upstream server. By default, the ``read-timeout`` of the upstream is used. | ``string`` | No | +|``send-timeout`` | The timeout for transmitting a request to an upstream server. By default, the ``send-timeout`` of the upstream is used. | ``string`` | No | +|``headers`` | The request headers used for health check requests. NGINX Plus always sets the ``Host`` , ``User-Agent`` and ``Connection`` headers for health check requests. | [[]header](#header) | No | +|``statusMatch`` | The expected response status codes of a health check. By default, the response should have status code 2xx or 3xx. Examples: ``"200"`` , ``"! 500"`` , ``"301-303 307"``. See the documentation of the [match](https://nginx.org/en/docs/http/ngx_http_upstream_hc_module.html?#match) directive. This not supported for gRPC type upstreams. | ``string`` | No | +|``grpcStatus`` | The expected [gRPC status code](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md#status-codes-and-their-use-in-grpc) of the upstream server response to the [Check method](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). Configure this field only if your gRPC services do not implement the gRPC health checking protocol. For example, configure ``12`` if the upstream server responds with `12 (UNIMPLEMENTED)` status code. Only valid on gRPC type upstreams. | ``int`` | No | +|``grpcService`` | The gRPC service to be monitored on the upstream server. Only valid on gRPC type upstreams. | ``string`` | No | +|``mandatory`` | Require every newly added server to pass all configured health checks before NGINX Plus sends traffic to it. If this is not specified, or is set to false, the server will be initially considered healthy. When combined with [slow-start](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#slow_start), it gives a new server more time to connect to databases and “warm up” before being asked to handle their full share of traffic. | ``bool`` | No | +|``persistent`` | Set the initial “up” state for a server after reload if the server was considered healthy before reload. Enabling persistent requires that the mandatory parameter is also set to `true`. | ``bool`` | No | +|``keepalive-time`` | Enables [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) connections for health checks and specifies the time during which requests can be processed through one keepalive connection. The default is ``60s``. | ``string`` | No | +{{}} + +### Upstream.SessionCookie + +The SessionCookie field configures session persistence which allows requests from the same client to be passed to the same upstream server. The information about the designated upstream server is passed in a session cookie generated by NGINX Plus. + +In the example below, we configure session persistence with a session cookie for an upstream and configure all the available parameters: + +```yaml +name: tea +service: tea-svc +port: 80 +sessionCookie: + enable: true + name: srv_id + path: / + expires: 1h + domain: .example.com + httpOnly: false + secure: true + samesite: strict +``` + +See the [`sticky`](https://nginx.org/en/docs/http/ngx_http_upstream_module.html?#sticky) directive for additional information. The session cookie corresponds to the `sticky cookie` method. + +Note: This feature is supported only in NGINX Plus. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables session persistence with a session cookie for an upstream server. The default is ``false``. | ``boolean`` | No | +|``name`` | The name of the cookie. | ``string`` | Yes | +|``path`` | The path for which the cookie is set. | ``string`` | No | +|``expires`` | The time for which a browser should keep the cookie. Can be set to the special value ``max`` , which will cause the cookie to expire on ``31 Dec 2037 23:55:55 GMT``. | ``string`` | No | +|``domain`` | The domain for which the cookie is set. | ``string`` | No | +|``httpOnly`` | Adds the ``HttpOnly`` attribute to the cookie. | ``boolean`` | No | +|``secure`` | Adds the ``Secure`` attribute to the cookie. | ``boolean`` | No | +|``samesite`` | Adds the ``SameSite`` attribute to the cookie. The allowed values are: ``strict``, ``lax``, ``none`` | ``string`` | No | +{{}} + +### Header + +The header defines an HTTP Header: + +```yaml +name: Host +value: example.com +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. | ``string`` | No | +{{}} + +### Action + +The action defines an action to perform for a request. + +In the example below, client requests are passed to an upstream `coffee`: + +```yaml + path: /coffee + action: + pass: coffee +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``pass`` | Passes requests to an upstream. The upstream with that name must be defined in the resource. | ``string`` | No | +|``redirect`` | Redirects requests to a provided URL. | [action.redirect](#actionredirect) | No | +|``return`` | Returns a preconfigured response. | [action.return](#actionreturn) | No | +|``proxy`` | Passes requests to an upstream with the ability to modify the request/response (for example, rewrite the URI or modify the headers). | [action.proxy](#actionproxy) | No | +{{}} + +\* -- an action must include exactly one of the following: `pass`, `redirect`, `return` or `proxy`. + +### Action.Redirect + +The redirect action defines a redirect to return for a request. + +In the example below, client requests are passed to a url `http://www.nginx.com`: + +```yaml +redirect: + url: http://www.nginx.com + code: 301 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``url`` | The URL to redirect the request to. Supported NGINX variables: ``$scheme`` , ``$http_x_forwarded_proto`` , ``$request_uri`` , ``$host``. Variables must be enclosed in curly braces. For example: ``${host}${request_uri}``. | ``string`` | Yes | +|``code`` | The status code of a redirect. The allowed values are: ``301`` , ``302`` , ``307`` , ``308``. The default is ``301``. | ``int`` | No | +{{}} + +### Action.Return + +The return action defines a preconfigured response for a request. + +In the example below, NGINX will respond with the preconfigured response for every request: + +```yaml +return: + code: 200 + type: text/plain + body: "Hello World\n" + headers: + - name: x-coffee + value: espresso +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``code`` | The status code of the response. The allowed values are: ``2XX``, ``4XX`` or ``5XX``. The default is ``200``. | ``int`` | No | +|``type`` | The MIME type of the response. The default is ``text/plain``. | ``string`` | No | +|``body`` | The body of the response. Supports NGINX variables*. Variables must be enclosed in curly brackets. For example: ``Request is ${request_uri}\n``. | ``string`` | Yes | +|``headers`` | The custom headers of the response. | [[]Action.Return.Header](#actionreturnheader) | No | +{{}} + +\* -- Supported NGINX variables: `$request_uri`, `$request_method`, `$request_body`, `$scheme`, `$http_`, `$args`, `$arg_`, `$cookie_`, `$host`, `$request_time`, `$request_length`, `$nginx_version`, `$pid`, `$connection`, `$remote_addr`, `$remote_port`, `$time_iso8601`, `$time_local`, `$server_addr`, `$server_port`, `$server_name`, `$server_protocol`, `$connections_active`, `$connections_reading`, `$connections_writing` and `$connections_waiting`. + +### Action.Return.Header + +The header defines an HTTP Header for a canned response in an actionReturn: + +```yaml +name: x-coffee +value: espresso +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. | ``string`` | Yes | +{{}} + +### Action.Proxy + +The proxy action passes requests to an upstream with the ability to modify the request/response (for example, rewrite the URI or modify the headers). + +In the example below, the request URI is rewritten to `/`, and the request and the response headers are modified: + +```yaml +proxy: + upstream: coffee + requestHeaders: + pass: true + set: + - name: My-Header + value: Value + - name: Client-Cert + value: ${ssl_client_escaped_cert} + responseHeaders: + add: + - name: My-Header + value: Value + - name: IC-Nginx-Version + value: ${nginx_version} + always: true + hide: + - x-internal-version + ignore: + - Expires + - Set-Cookie + pass: + - Server + rewritePath: / +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``upstream`` | The name of the upstream which the requests will be proxied to. The upstream with that name must be defined in the resource. | ``string`` | Yes | +|``requestHeaders`` | The request headers modifications. | [action.Proxy.RequestHeaders](#actionproxyrequestheaders) | No | +|``responseHeaders`` | The response headers modifications. | [action.Proxy.ResponseHeaders](#actionproxyresponseheaders) | No | +|``rewritePath`` | The rewritten URI. If the route path is a regular expression -- starts with `~` -- the `rewritePath` can include capture groups with ``$1-9``. For example `$1` for the first group, and so on. For more information, check the [rewrite](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/rewrites) example. | ``string`` | No | +{{}} + +### Action.Proxy.RequestHeaders + +The RequestHeaders field modifies the headers of the request to the proxied upstream server. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``pass`` | Passes the original request headers to the proxied upstream server. See the [proxy_pass_request_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_request_headers) directive for more information. Default is true. | ``bool`` | No | +|``set`` | Allows redefining or appending fields to present request headers passed to the proxied upstream servers. See the [proxy_set_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) directive for more information. | [[]header](#actionproxyrequestheaderssetheader) | No | +{{}} + +### Action.Proxy.RequestHeaders.Set.Header + +The header defines an HTTP Header: + +```yaml +name: My-Header +value: My-Value +``` + +It is possible to override the default value of the `Host` header, which NGINX Ingress Controller sets to [`$host`](https://nginx.org/en/docs/http/ngx_http_core_module.html#var_host): + +```yaml +name: Host +value: example.com +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. Supports NGINX variables*. Variables must be enclosed in curly brackets. For example: ``${scheme}``. | ``string`` | No | +{{}} + +\* -- Supported NGINX variables: `$request_uri`, `$request_method`, `$request_body`, `$scheme`, `$http_`, `$args`, `$arg_`, `$cookie_`, `$host`, `$request_time`, `$request_length`, `$nginx_version`, `$pid`, `$connection`, `$remote_addr`, `$remote_port`, `$time_iso8601`, `$time_local`, `$server_addr`, `$server_port`, `$server_name`, `$server_protocol`, `$connections_active`, `$connections_reading`, `$connections_writing`, `$connections_waiting`, `$ssl_cipher`, `$ssl_ciphers`, `$ssl_client_cert`, `$ssl_client_escaped_cert`, `$ssl_client_fingerprint`, `$ssl_client_i_dn`, `$ssl_client_i_dn_legacy`, `$ssl_client_raw_cert`, `$ssl_client_s_dn`, `$ssl_client_s_dn_legacy`, `$ssl_client_serial`, `$ssl_client_v_end`, `$ssl_client_v_remain`, `$ssl_client_v_start`, `$ssl_client_verify`, `$ssl_curves`, `$ssl_early_data`, `$ssl_protocol`, `$ssl_server_name`, `$ssl_session_id`, `$ssl_session_reused`, `$jwt_claim_` (NGINX Plus only) and `$jwt_header_` (NGINX Plus only). + +### Action.Proxy.ResponseHeaders + +The ResponseHeaders field modifies the headers of the response to the client. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``hide`` | The headers that will not be passed* in the response to the client from a proxied upstream server. See the [proxy_hide_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directive for more information. | ``[]string`` | No | +|``pass`` | Allows passing the hidden header fields* to the client from a proxied upstream server. See the [proxy_pass_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directive for more information. | ``[]string`` | No | +|``ignore`` | Disables processing of certain headers** to the client from a proxied upstream server. See the [proxy_ignore_headers](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ignore_headers) directive for more information. | ``[]string`` | No | +|``add`` | Adds headers to the response to the client. | [[]addHeader](#addheader) | No | +{{}} + +\* -- Default hidden headers are: `Date`, `Server`, `X-Pad` and `X-Accel-...`. + +\** -- The following fields can be ignored: `X-Accel-Redirect`, `X-Accel-Expires`, `X-Accel-Limit-Rate`, `X-Accel-Buffering`, `X-Accel-Charset`, `Expires`, `Cache-Control`, `Set-Cookie` and `Vary`. + +### AddHeader + +The addHeader defines an HTTP Header with an optional `always` field: + +```yaml +name: My-Header +value: My-Value +always: true +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. Supports NGINX variables*. Variables must be enclosed in curly brackets. For example: ``${scheme}``. | ``string`` | No | +|``always`` | If set to true, add the header regardless of the response status code**. Default is false. See the [add_header](http://nginx.org/en/docs/http/ngx_http_headers_module.html#add_header) directive for more information. | ``bool`` | No | +{{}} + +\* -- Supported NGINX variables: `$request_uri`, `$request_method`, `$request_body`, `$scheme`, `$http_`, `$args`, `$arg_`, `$cookie_`, `$host`, `$request_time`, `$request_length`, `$nginx_version`, `$pid`, `$connection`, `$remote_addr`, `$remote_port`, `$time_iso8601`, `$time_local`, `$server_addr`, `$server_port`, `$server_name`, `$server_protocol`, `$connections_active`, `$connections_reading`, `$connections_writing`, `$connections_waiting`, `$ssl_cipher`, `$ssl_ciphers`, `$ssl_client_cert`, `$ssl_client_escaped_cert`, `$ssl_client_fingerprint`, `$ssl_client_i_dn`, `$ssl_client_i_dn_legacy`, `$ssl_client_raw_cert`, `$ssl_client_s_dn`, `$ssl_client_s_dn_legacy`, `$ssl_client_serial`, `$ssl_client_v_end`, `$ssl_client_v_remain`, `$ssl_client_v_start`, `$ssl_client_verify`, `$ssl_curves`, `$ssl_early_data`, `$ssl_protocol`, `$ssl_server_name`, `$ssl_session_id`, `$ssl_session_reused`, `$jwt_claim_` (NGINX Plus only) and `$jwt_header_` (NGINX Plus only). + +{{< note >}} If `always` is false, the response header is added only if the response status code is any of `200`, `201`, `204`, `206`, `301`, `302`, `303`, `304`, `307` or `308`. {{< /note >}} + +### Split + +The split defines a weight for an action as part of the splits configuration. + +In the example below NGINX passes 80% of requests to the upstream `coffee-v1` and the remaining 20% to `coffee-v2`: + +```yaml +splits: +- weight: 80 + action: + pass: coffee-v1 +- weight: 20 + action: + pass: coffee-v2 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``weight`` | The weight of an action. Must fall into the range ``0..100``. The sum of the weights of all splits must be equal to ``100``. | ``int`` | Yes | +|``action`` | The action to perform for a request. | [action](#action) | Yes | +{{}} + +### Match + +The match defines a match between conditions and an action or splits. + +In the example below, NGINX routes requests with the path `/coffee` to different upstreams based on the value of the cookie `user`: + +- `user=john` -> `coffee-future` +- `user=bob` -> `coffee-deprecated` +- If the cookie is not set or not equal to either `john` or `bob`, NGINX routes to `coffee-stable` + +```yaml +path: /coffee +matches: +- conditions: + - cookie: user + value: john + action: + pass: coffee-future +- conditions: + - cookie: user + value: bob + action: + pass: coffee-deprecated +action: + pass: coffee-stable +``` + +In the next example, NGINX routes requests based on the value of the built-in [`$request_method` variable](https://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_method), which represents the HTTP method of a request: + +- all POST requests -> `coffee-post` +- all non-POST requests -> `coffee` + +```yaml +path: /coffee +matches: +- conditions: + - variable: $request_method + value: POST + action: + pass: coffee-post +action: + pass: coffee +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``conditions`` | A list of conditions. Must include at least 1 condition. | [[]condition](#condition) | Yes | +|``action`` | The action to perform for a request. | [action](#action) | No | +|``splits`` | The splits configuration for traffic splitting. Must include at least 2 splits. | [[]split](#split) | No | +{{}} + +{{< note >}} A match must include exactly one of the following: `action` or `splits`. {{< /note >}} + +### Condition + +The condition defines a condition in a match. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``header`` | The name of a header. Must consist of alphanumeric characters or ``-``. | ``string`` | No | +|``cookie`` | The name of a cookie. Must consist of alphanumeric characters or ``_``. | ``string`` | No | +|``argument`` | The name of an argument. Must consist of alphanumeric characters or ``_``. | ``string`` | No | +|``variable`` | The name of an NGINX variable. Must start with ``$``. See the list of the supported variables below the table. | ``string`` | No | +|``value`` | The value to match the condition against. How to define a value is shown below the table. | ``string`` | Yes | +{{}} + +{{< note >}} a condition must include exactly one of the following: `header`, `cookie`, `argument` or `variable`. {{< /note >}} + +Supported NGINX variables: `$args`, `$http2`, `$https`, `$remote_addr`, `$remote_port`, `$query_string`, `$request`, `$request_body`, `$request_uri`, `$request_method`, `$scheme`. Find the documentation for each variable [here](https://nginx.org/en/docs/varindex.html). + +The value supports two kinds of matching: + +- *Case-insensitive string comparison*. For example: + - `john` -- case-insensitive matching that succeeds for strings, such as `john`, `John`, `JOHN`. + - `!john` -- negation of the case-insensitive matching for john that succeeds for strings, such as `bob`, `anything`, `''` (empty string). +- *Matching with a regular expression*. Note that NGINX supports regular expressions compatible with those used by the Perl programming language (PCRE). For example: + - `~^yes` -- a case-sensitive regular expression that matches any string that starts with `yes`. For example: `yes`, `yes123`. + - `!~^yes` -- negation of the previous regular expression that succeeds for strings like `YES`, `Yes123`, `noyes`. (The negation mechanism is not part of the PCRE syntax). + - `~*no$` -- a case-insensitive regular expression that matches any string that ends with `no`. For example: `no`, `123no`, `123NO`. + +{{< note >}} A value must not include any unescaped double quotes (`"`) and must not end with an unescaped backslash (`\`). For example, the following are invalid values: `some"value`, `somevalue\`. {{< /note >}} + +### ErrorPage + +The errorPage defines a custom response for a route for the case when either an upstream server responds with (or NGINX generates) an error status code. The custom response can be a redirect or a canned response. See the [error_page](https://nginx.org/en/docs/http/ngx_http_core_module.html#error_page) directive for more information. + +```yaml +path: /coffee +errorPages: +- codes: [502, 503] + redirect: + code: 301 + url: https://nginx.org +- codes: [404] + return: + code: 200 + body: "Original resource not found, but success!" +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``codes`` | A list of error status codes. | ``[]int`` | Yes | +|``redirect`` | The redirect action for the given status codes. | [errorPage.Redirect](#errorpageredirect) | No | +|``return`` | The canned response action for the given status codes. | [errorPage.Return](#errorpagereturn) | No | +{{}} + +{{< note >}} An errorPage must include exactly one of the following: `return` or `redirect`. {{< /note >}} + +### ErrorPage.Redirect + +The redirect defines a redirect for an errorPage. + +In the example below, NGINX responds with a redirect when a response from an upstream server has a 404 status code. + +```yaml +codes: [404] +redirect: + code: 301 + url: ${scheme}://cafe.example.com/error.html +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``code`` | The status code of a redirect. The allowed values are: ``301`` , ``302`` , ``307`` , ``308``. The default is ``301``. | ``int`` | No | +|``url`` | The URL to redirect the request to. Supported NGINX variables: ``$scheme`` and ``$http_x_forwarded_proto``. Variables must be enclosed in curly braces. For example: ``${scheme}``. | ``string`` | Yes | +{{}} + +### ErrorPage.Return + +The return defines a canned response for an errorPage. + +In the example below, NGINX responds with a canned response when a response from an upstream server has either 401 or 403 status code. + +```yaml +codes: [401, 403] +return: + code: 200 + type: application/json + body: | + {\"msg\": \"You don't have permission to do this\"} + headers: + - name: x-debug-original-statuses + value: ${upstream_status} +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``code`` | The status code of the response. The default is the status code of the original response. | ``int`` | No | +|``type`` | The MIME type of the response. The default is ``text/html``. | ``string`` | No | +|``body`` | The body of the response. Supported NGINX variable: ``$upstream_status`` . Variables must be enclosed in curly braces. For example: ``${upstream_status}``. | ``string`` | Yes | +|``headers`` | The custom headers of the response. | [[]errorPage.Return.Header](#errorpagereturnheader) | No | +{{}} + +### ErrorPage.Return.Header + +The header defines an HTTP Header for a canned response in an errorPage: + +```yaml +name: x-debug-original-statuses +value: ${upstream_status} +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. Supported NGINX variable: ``$upstream_status`` . Variables must be enclosed in curly braces. For example: ``${upstream_status}``. | ``string`` | No | +{{}} + +## Using VirtualServer and VirtualServerRoute + +You can use the usual `kubectl` commands to work with VirtualServer and VirtualServerRoute resources, similar to Ingress resources. + +For example, the following command creates a VirtualServer resource defined in `cafe-virtual-server.yaml` with the name `cafe`: + +```shell +kubectl apply -f cafe-virtual-server.yaml +``` +```text +virtualserver.k8s.nginx.org "cafe" created +``` + +You can get the resource by running: + +```shell +kubectl get virtualserver cafe +``` +```text +NAME STATE HOST IP PORTS AGE +cafe Valid cafe.example.com 12.13.23.123 [80,443] 3m +``` + +In `kubectl get` and similar commands, you can use the short name `vs` instead of `virtualserver`. + +Similarly, for VirtualServerRoute you can use `virtualserverroute` or the short name `vsr`. + +### Using Snippets + +Snippets allow you to insert raw NGINX config into different contexts of NGINX configuration. In the example below, we use snippets to configure several NGINX features in a VirtualServer: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: cafe + namespace: cafe +spec: + http-snippets: | + limit_req_zone $binary_remote_addr zone=mylimit:10m rate=1r/s; + proxy_cache_path /tmp keys_zone=one:10m; + host: cafe.example.com + tls: + secret: cafe-secret + server-snippets: | + limit_req zone=mylimit burst=20; + upstreams: + - name: tea + service: tea-svc + port: 80 + - name: coffee + service: coffee-svc + port: 80 + routes: + - path: /tea + location-snippets: | + proxy_cache one; + proxy_cache_valid 200 10m; + action: + pass: tea + - path: /coffee + action: + pass: coffee +``` + +For additional information, view the [Advanced configuration with Snippets]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md" >}}) topic. + +### Validation + +Two types of validation are available for VirtualServer and VirtualServerRoute resources: + +- *Structural validation* by the `kubectl` and Kubernetes API server. +- *Comprehensive validation* by NGINX Ingress Controller. + +#### Structural Validation + +The custom resource definitions for VirtualServer and VirtualServerRoute include structural OpenAPI schema which describes the type of every field of those resources. + +If you try to create (or update) a resource that violates the structural schema (for example, you use a string value for the port field of an upstream), `kubectl` and Kubernetes API server will reject such a resource: + +- Example of `kubectl` validation: + + ```shell + kubectl apply -f cafe-virtual-server.yaml + ``` + ```text + error: error validating "cafe-virtual-server.yaml": error validating data: ValidationError(VirtualServer.spec.upstreams[0].port): invalid type for org.nginx.k8s.v1.VirtualServer.spec.upstreams.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + ``` + +- Example of Kubernetes API server validation: + + ```shell + kubectl apply -f cafe-virtual-server.yaml --validate=false + ``` + ```text + The VirtualServer "cafe" is invalid: []: Invalid value: map[string]interface {}{ ... }: validation failure list: + spec.upstreams.port in body must be of type integer: "string" + ``` + +If a resource is not rejected (it doesn't violate the structural schema), NGINX Ingress Controller will validate it further. + +#### Comprehensive Validation + +NGINX Ingress Controller validates the fields of the VirtualServer and VirtualServerRoute resources. If a resource is invalid, NGINX Ingress Controller will reject it: the resource will continue to exist in the cluster, but NGINX Ingress Controller will ignore it. + +You can check if NGINX Ingress Controller successfully applied the configuration for a VirtualServer. For our example `cafe` VirtualServer, we can run: + +```shell +kubectl describe vs cafe +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 16s nginx-ingress-controller Configuration for default/cafe was added or updated +``` + +Note how the events section includes a Normal event with the AddedOrUpdated reason that informs us that the configuration was successfully applied. + +If you create an invalid resource, NGINX Ingress Controller will reject it and emit a Rejected event. For example, if you create a VirtualServer `cafe` with two upstream with the same name `tea`, you will get: + +```shell +kubectl describe vs cafe +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 12s nginx-ingress-controller VirtualServer default/cafe is invalid and was rejected: spec.upstreams[1].name: Duplicate value: "tea" +``` + +Note how the events section includes a Warning event with the Rejected reason. + +Additionally, this information is also available in the `status` field of the VirtualServer resource. Note the Status section of the VirtualServer: + +```shell +kubectl describe vs cafe +``` +```text +... +Status: + External Endpoints: + Ip: 12.13.23.123 + Ports: [80,443] + Message: VirtualServer default/cafe is invalid and was rejected: spec.upstreams[1].name: Duplicate value: "tea" + Reason: Rejected + State: Invalid +``` + +NGINX Ingress Controller validates VirtualServerRoute resources in a similar way. + +**Note**: If you make an existing resource invalid, NGINX Ingress Controller will reject it and remove the corresponding configuration from NGINX. + +## Customization using ConfigMap + +You can customize the NGINX configuration for VirtualServer and VirtualServerRoutes resources using the [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource). Most of the ConfigMap keys are supported, with the following exceptions: + +- `proxy-hide-headers` +- `proxy-pass-headers` +- `hsts` +- `hsts-max-age` +- `hsts-include-subdomains` +- `hsts-behind-proxy` +- `redirect-to-https` +- `ssl-redirect` diff --git a/content/includes/nic/installation/create-common-resources.md b/content/includes/nic/installation/create-common-resources.md new file mode 100644 index 000000000..7cfe90e48 --- /dev/null +++ b/content/includes/nic/installation/create-common-resources.md @@ -0,0 +1,27 @@ +--- +docs: DOCS-1464 +--- + +In this section, you'll create resources that most NGINX Ingress Controller installations require: + +1. (Optional) Create a secret for the default NGINX server's TLS certificate and key. Complete this step only if you're using the [default server TLS secret]({{< ref "/nic/configuration/global-configuration/command-line-arguments#cmdoption-default-server-tls-secret.md" >}}) command-line argument. If you're not, feel free to skip this step. + + By default, the server returns a _404 Not Found_ page for all requests when no ingress rules are set up. Although we provide a self-signed certificate and key for testing purposes, we recommend using your own certificate. + + ```shell + kubectl apply -f examples/shared-examples/default-server-secret/default-server-secret.yaml + ``` + +2. Create a ConfigMap to customize your NGINX settings: + + ```shell + kubectl apply -f deployments/common/nginx-config.yaml + ``` + +3. Create an `IngressClass` resource. NGINX Ingress Controller won't start without an `IngressClass` resource. + + ```shell + kubectl apply -f deployments/common/ingress-class.yaml + ``` + + If you want to make this NGINX Ingress Controller instance your cluster's default, uncomment the `ingressclass.kubernetes.io/is-default-class` annotation. This action will auto-assign `IngressClass` to new ingresses that don't specify an `ingressClassName`. diff --git a/content/includes/nic/installation/create-custom-resources.md b/content/includes/nic/installation/create-custom-resources.md new file mode 100644 index 000000000..d7ad9ec27 --- /dev/null +++ b/content/includes/nic/installation/create-custom-resources.md @@ -0,0 +1,49 @@ +--- +docs: DOCS-1463 +--- + +To make sure your NGINX Ingress Controller pods reach the `Ready` state, you'll need to create custom resource definitions (CRDs) for various components. + +Alternatively, you can disable this requirement by setting the `-enable-custom-resources` command-line argument to `false`. + +There are two ways you can install the custom resource definitions: + +1. Using a URL to apply a single CRD yaml file, which we recommend. +1. Applying your local copy of the CRD yaml files, which requires you to clone the repository. + +The core custom CRDs are the following: + +- [VirtualServer and VirtualServerRoute]({{< ref "/nic/configuration/virtualserver-and-virtualserverroute-resources.md" >}}) +- [TransportServer]({{< ref "/nic/configuration/transportserver-resource.md" >}}) +- [Policy]({{< ref "/nic/configuration/policy-resource.md" >}}) +- [GlobalConfiguration]({{< ref "/nic/configuration/global-configuration/globalconfiguration-resource.md" >}}) + +{{}} + +{{%tab name="Install CRDs from single YAML"%}} + +```shell +kubectl apply -f https://raw.githubusercontent.com/nginx/kubernetes-ingress/v{{< nic-version >}}/deploy/crds.yaml +``` + +{{%/tab%}} + +{{%tab name="Install CRDs after cloning the repo"%}} + +{{< note >}} If you are installing the CRDs this way, ensure you have first cloned the repository. {{< /note >}} + +{{< note >}} Please make sure to read the steps outlined in [Upgrade to V4](https://docs.nginx.com/nginx-ingress-controller/installation/installing-nic/upgrade-to-v4/#update-custom-resource-apiversion) before running the CRD upgrade and perform the steps if applicable. +{{< /note >}} + + +```shell +kubectl apply -f config/crd/bases/k8s.nginx.org_virtualservers.yaml +kubectl apply -f config/crd/bases/k8s.nginx.org_virtualserverroutes.yaml +kubectl apply -f config/crd/bases/k8s.nginx.org_transportservers.yaml +kubectl apply -f config/crd/bases/k8s.nginx.org_policies.yaml +kubectl apply -f config/crd/bases/k8s.nginx.org_globalconfigurations.yaml +``` + +{{%/tab%}} + +{{}} diff --git a/content/includes/nic/installation/deploy-controller.md b/content/includes/nic/installation/deploy-controller.md new file mode 100644 index 000000000..5723c73ae --- /dev/null +++ b/content/includes/nic/installation/deploy-controller.md @@ -0,0 +1,10 @@ +--- +docs: DOCS-1462 +--- + +You have two options for deploying NGINX Ingress Controller: + +- **Deployment**. Choose this method for the flexibility to dynamically change the number of NGINX Ingress Controller replicas. +- **DaemonSet**. Choose this method if you want NGINX Ingress Controller to run on all nodes or a subset of nodes. + +Before you start, update the [command-line arguments]({{< ref "/nic/configuration/global-configuration/command-line-arguments.md" >}}) for the NGINX Ingress Controller container in the relevant manifest file to meet your specific requirements. diff --git a/content/includes/nic/installation/download-jwt.md b/content/includes/nic/installation/download-jwt.md new file mode 100644 index 000000000..d89c65a43 --- /dev/null +++ b/content/includes/nic/installation/download-jwt.md @@ -0,0 +1,10 @@ +--- +docs: "DOCS-000" +--- + +1. Log in to [MyF5](https://my.f5.com/manage/s/). +2. Go to **My Products & Plans > Subscriptions** to see your active subscriptions. +3. Find your NGINX products or services subscription, and select the **Subscription ID** for details. +4. Download the **JSON Web Token (JWT)** from the subscription page. + +{{< note >}} The Connectivity Stack for Kubernetes JWT does not work with NGINX Plus reporting. A regular NGINX Plus instance JWT must be used. {{< /note >}} diff --git a/content/includes/nic/installation/jwt-password-note.md b/content/includes/nic/installation/jwt-password-note.md new file mode 100644 index 000000000..147c67af3 --- /dev/null +++ b/content/includes/nic/installation/jwt-password-note.md @@ -0,0 +1,11 @@ +--- +docs: +--- + +{{< note >}} For security, follow these practices with JSON Web Tokens (JWTs), passwords, and shell history: + +1. **JWTs:** JWTs are sensitive information. Store them securely. Delete them after use to prevent unauthorized access. + +1. **Shell history:** Commands that include JWTs or passwords are recorded in the history of your shell, in plain text. Clear your shell history after running such commands. For example, if you use bash, you can delete commands in your `~/.bash_history` file. Alternatively, you can run the `history -c` command to erase your shell history. + +Follow these practices to help ensure the security of your system and data. {{< /note >}} diff --git a/content/includes/nic/installation/manifests/daemonset.md b/content/includes/nic/installation/manifests/daemonset.md new file mode 100644 index 000000000..c19376eb2 --- /dev/null +++ b/content/includes/nic/installation/manifests/daemonset.md @@ -0,0 +1,21 @@ +--- +docs: DOCS-1465 +--- + +For additional context on managing containers using Kubernetes DaemonSets, refer to the official Kubernetes [DaemonSets](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) documentation. + +When you deploy NGINX Ingress Controller as a DaemonSet, Kubernetes creates an Ingress Controller pod on every node in the cluster. + +- For NGINX, run: + + ```shell + kubectl apply -f deployments/daemon-set/nginx-ingress.yaml + ``` + +- For NGINX Plus, run: + + ```shell + kubectl apply -f deployments/daemon-set/nginx-plus-ingress.yaml + ``` + + Update the `nginx-plus-ingress.yaml` file to include your chosen image from the F5 Container registry or your custom container image. diff --git a/content/includes/nic/installation/manifests/deployment.md b/content/includes/nic/installation/manifests/deployment.md new file mode 100644 index 000000000..3786a2256 --- /dev/null +++ b/content/includes/nic/installation/manifests/deployment.md @@ -0,0 +1,21 @@ +--- +docs: DOCS-1467 +--- + +For additional context on managing containers using Kubernetes Deployments, refer to the official Kubernetes [Deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) documentation. + +When you deploy NGINX Ingress Controller as a Deployment, Kubernetes automatically sets up a single NGINX Ingress Controller pod. + +- For NGINX, run: + + ```shell + kubectl apply -f deployments/deployment/nginx-ingress.yaml + ``` + +- For NGINX Plus, run: + + ```shell + kubectl apply -f deployments/deployment/nginx-plus-ingress.yaml + ``` + + Update the `nginx-plus-ingress.yaml` file to include your chosen image from the F5 Container registry or your custom container image. diff --git a/content/includes/nic/installation/manifests/verify-pods-are-running.md b/content/includes/nic/installation/manifests/verify-pods-are-running.md new file mode 100644 index 000000000..9f11fa7b0 --- /dev/null +++ b/content/includes/nic/installation/manifests/verify-pods-are-running.md @@ -0,0 +1,10 @@ +--- +docs: DOCS-1466 +--- + + +To confirm the NGINX Ingress Controller pods are operational, run: + +```shell +kubectl get pods --namespace=nginx-ingress +``` diff --git a/content/includes/nic/rbac/set-up-rbac.md b/content/includes/nic/rbac/set-up-rbac.md new file mode 100644 index 000000000..a8bd5a176 --- /dev/null +++ b/content/includes/nic/rbac/set-up-rbac.md @@ -0,0 +1,33 @@ +--- +docs: DOCS-1468 +--- + +{{}}To complete these steps you need admin access to your cluster. Refer to to your Kubernetes platform's documentation to set up admin access. For Google Kubernetes Engine (GKE), you can refer to their [Role-Based Access Control guide](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control).{{}} + +1. Create a namespace and a service account: + + ```shell + kubectl apply -f deployments/common/ns-and-sa.yaml + ``` + +2. Create a cluster role and binding for the service account: + + ```shell + kubectl apply -f deployments/rbac/rbac.yaml + ``` + +
+ +If you're planning to use NGINX App Protect or NGINX App Protect DoS, additional roles and bindings are needed. + +1. (NGINX App Protect only) Create the *App Protect* role and binding: + + ```shell + kubectl apply -f deployments/rbac/ap-rbac.yaml + ``` + +2. (NGINX App Protect DoS only) Create the *App Protect DoS* role and binding: + + ```shell + kubectl apply -f deployments/rbac/apdos-rbac.yaml + ``` diff --git a/content/nic/_index.md b/content/nic/_index.md new file mode 100644 index 000000000..8d93ce681 --- /dev/null +++ b/content/nic/_index.md @@ -0,0 +1,5 @@ +--- +title: NGINX Ingress Controller +menu: docs +url: /nginx-ingress-controller/ +--- diff --git a/content/nic/community.md b/content/nic/community.md new file mode 100644 index 000000000..4a10a6083 --- /dev/null +++ b/content/nic/community.md @@ -0,0 +1,23 @@ +--- +title: Community and contributing +weight: 2200 +nd-content-type: reference +nd-product: NIC +nd-docs: DOCS-1447 +--- + +There are a few ways to get involved with the F5 NGINX Ingress Controller community and contribute to the project. + +# Community + +- Our [GitHub issues page](https://github.com/nginx/kubernetes-ingress/issues) or [GitHub discussions page](https://github.com/nginx/kubernetes-ingress/discussions) offers more space for an asynchronous technical discussion. + +- You can can also get help through the [NGINX Community Forum](https://community.nginx.org/). + +# Contribute + +Please see our [contributing guide](https://github.com/nginx/kubernetes-ingress/blob/main/CONTRIBUTING.md) to get involved with code or documentation. + +# License + +[Apache License, Version 2.0](https://github.com/nginx/kubernetes-ingress/blob/main/LICENSE) diff --git a/content/nic/configuration/_index.md b/content/nic/configuration/_index.md new file mode 100644 index 000000000..5554b2f83 --- /dev/null +++ b/content/nic/configuration/_index.md @@ -0,0 +1,4 @@ +--- +title: Configuration +weight: 1400 +--- diff --git a/content/nic/configuration/access-control.md b/content/nic/configuration/access-control.md new file mode 100644 index 000000000..54fee5f91 --- /dev/null +++ b/content/nic/configuration/access-control.md @@ -0,0 +1,120 @@ +--- +title: Deploy a Policy for access control +weight: 900 +toc: true +docs: DOCS-000 +--- + +This topic describes how to use F5 NGINX Ingress Controller to apply and update a Policy for access control. It demonstrates it using an example application and a [VirtualServer custom resource]({{< ref "/nic/configuration/virtualserver-and-virtualserverroute-resources.md" >}}). + +--- + +## Before you begin + +You should have a [working NGINX Ingress Controller]({{< ref "/nic/installation/installing-nic/installation-with-helm.md" >}}) instance. + +For ease of use in shell commands, set two shell variables: + +1. The public IP address for your NGINX Ingress Controller instance. + +```shell +IC_IP= +``` + +2. The HTTP port of the same instance. + +```shell +IC_HTTP_PORT= +``` + +--- + +## Deploy the example application + +Create the file _webapp.yaml_ with the following contents: + +{{< ghcode "https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/custom-resources/access-control/webapp.yaml" >}} + +Apply it using `kubectl`: + +```shell +kubectl apply -f webapp.yaml +``` + +--- + +## Deploy a Policy to create a deny rule + +Create a file named _access-control-policy-deny.yaml_. The highlighted _deny_ field will be used by the example application, and should be changed to the subnet of your machine. + +{{< ghcode "https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/custom-resources/access-control/access-control-policy-deny.yaml" "hl_lines=7-8" >}} + +Apply the policy: + +```shell +kubectl apply -f access-control-policy-deny.yaml +``` + +--- + +## Configure load balancing + +Create a file named _virtual-server.yaml_ for the VirtualServer resource. The _policies_ field references the access control Policy created in the previous section. + +{{< ghcode "https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/custom-resources/access-control/virtual-server.yaml" "hl_lines=7-8" >}} + +Apply the policy: + +```shell +kubectl apply -f virtual-server.yaml +``` + +--- + +## Test the example application + +Use `curl` to attempt to access the application: + +```shell +curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP http://webapp.example.com:$IC_HTTP_PORT +``` +```text + +403 Forbidden + +

403 Forbidden

+ + +``` + +The *403* response is expected, successfully blocking your machine. + +--- + +## Update the Policy to create an allow rule + +Update the Policy with the file _access-control-policy-allow.yaml_, setting the _allow_ field to the subnet of your machine. + +{{< ghcode "https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/custom-resources/access-control/access-control-policy-allow.yaml" "hl_lines=7-8" >}} + +Apply the Policy: + +```shell +kubectl apply -f access-control-policy-allow.yaml +``` + +---- + +## Verify the Policy update + +Attempt to access the application again: + +```shell +curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP http://webapp.example.com:$IC_HTTP_PORT +``` +```text +Server address: 10.64.0.13:8080 +Server name: webapp-5cbbc7bd78-wf85w +``` + +The successful response demonstrates that the policy has been updated. diff --git a/content/nic/configuration/configuration-examples.md b/content/nic/configuration/configuration-examples.md new file mode 100644 index 000000000..0a0e6e3bb --- /dev/null +++ b/content/nic/configuration/configuration-examples.md @@ -0,0 +1,13 @@ +--- +docs: DOCS-584 +doctypes: +- '' +title: Configuration examples +toc: true +weight: 400 +--- + +Our [GitHub repo](https://github.com/nginx/kubernetes-ingress) includes a number of configuration examples: + +- [*Examples of Custom Resources*](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources) show how to advanced NGINX features by using VirtualServer, VirtualServerRoute, TransportServer and Policy Custom Resources. +- [*Examples of Ingress Resources*](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources) show how to use advanced NGINX features in Ingress resources with annotations. diff --git a/content/nic/configuration/global-configuration/_index.md b/content/nic/configuration/global-configuration/_index.md new file mode 100644 index 000000000..4f71e2f5f --- /dev/null +++ b/content/nic/configuration/global-configuration/_index.md @@ -0,0 +1,8 @@ +--- +title: Global configuration +description: +weight: 100 +menu: + docs: + parent: NGINX Ingress Controller +--- diff --git a/content/nic/configuration/global-configuration/command-line-arguments.md b/content/nic/configuration/global-configuration/command-line-arguments.md new file mode 100644 index 000000000..79d8f3f02 --- /dev/null +++ b/content/nic/configuration/global-configuration/command-line-arguments.md @@ -0,0 +1,685 @@ +--- +docs: DOCS-585 +doctypes: +- '' +title: Command-line arguments +toc: true +weight: 100 +--- + +F5 NGINX Ingress Controller supports several command-line arguments, which are set based on installation method: + +- If you're using *Kubernetes Manifests* to install NGINX Ingress Controller, modify the Manifests to set the command-line arguments. View the [Installation with Manifests]({{}}) topic for more information. +- If you're using *Helm* to install NGINX Ingress Controller, modify the parameters of the Helm chart to set the command-line arguments. View the [Installation with Helm]({{}}) topic for more information. + + + +### -enable-snippets + +Enable custom NGINX configuration snippets in Ingress, VirtualServer, VirtualServerRoute and TransportServer resources. + +Default `false`. + + + +--- + +### -default-server-tls-secret `` + +Secret with a TLS certificate and key for TLS termination of the default server. + +- If not set, certificate and key in the file `/etc/nginx/secrets/default` are used. +- If `/etc/nginx/secrets/default` doesn't exist, NGINX Ingress Controller will configure NGINX to reject TLS connections to the default server. +- If a secret is set, but NGINX Ingress Controller is not able to fetch it from Kubernetes API, or it is not set and NGINX Ingress Controller fails to read the file "/etc/nginx/secrets/default", NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -wildcard-tls-secret `` + +A Secret with a TLS certificate and key for TLS termination of every Ingress/VirtualServer host for which TLS termination is enabled but the Secret is not specified. + +- If the argument is not set, for such Ingress/VirtualServer hosts NGINX will break any attempt to establish a TLS connection +- If the argument is set, but NGINX Ingress Controller is not able to fetch the Secret from Kubernetes API, NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -enable-custom-resources + +Enables custom resources. + +Default `true`. + + + + +--- + +### -enable-oidc + +Enables OIDC policies. + +Default `false`. + + + +--- + +### -enable-leader-election + +Enables Leader election to avoid multiple replicas of the controller reporting the status of Ingress, VirtualServer and VirtualServerRoute resources -- only one replica will report status. +Default `true`. + +See [-report-ingress-status](#cmdoption-report-ingress-status) flag. + + + +--- + +### -enable-tls-passthrough + +Enable TLS Passthrough on port 443. + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + + +--- + +### -tls-passthrough-port `` + +Set the port for TLS Passthrough. +Format: `[1024 - 65535]` (default `443`) + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + + +--- + +### -enable-cert-manager + +Enable x509 automated certificate management for VirtualServer resources using cert-manager (cert-manager.io). + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + + +--- + +### -enable-external-dns + +Enable integration with ExternalDNS for configuring public DNS entries for VirtualServer resources using [ExternalDNS](https://github.com/kubernetes-sigs/external-dns). + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + +--- + +### -external-service `` + +Specifies the name of the service with the type LoadBalancer through which the NGINX Ingress Controller pods are exposed externally. The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. + +For Ingress resources only: Requires [-report-ingress-status](#cmdoption-report-ingress-status). + + + +--- + +### -ingresslink `` + +Specifies the name of the IngressLink resource, which exposes the NGINX Ingress Controller pods via a BIG-IP system. The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. + +For Ingress resources only: Requires [-report-ingress-status](#cmdoption-report-ingress-status). + + + +--- + +### -global-configuration `` + +A GlobalConfiguration resource for global configuration of NGINX Ingress Controller. + +Format: `/` + +Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). + + + +--- + +### -health-status + +Adds a location "/nginx-health" to the default server. The location responds with the 200 status code for any request. + +Useful for external health-checking of NGINX Ingress Controller. + + + +--- + +### -health-status-uri `` + +Sets the URI of health status location in the default server. Requires [-health-status](#cmdoption-health-status). (default `/nginx-health`) + + + +--- + +### -ingress-class `` + +The `-ingress-class` argument refers to the name of the resource `kind: IngressClass`. An IngressClass resource with a name equal to the class must be deployed. Otherwise, NGINX Ingress Controller will fail to start. +NGINX Ingress Controller will only process Ingress resources that belong to its class (Whose `ingressClassName` value matches the value of `-ingress-class`), skipping the ones without it. It will also process all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the `ingressClassName` field. + +Default `nginx`. + + + +--- + +### -ingress-template-path `` + +Path to the ingress NGINX configuration template for an ingress resource. Default for NGINX is `nginx.ingress.tmpl`; default for NGINX Plus is `nginx-plus.ingress.tmpl`. + + + +--- + +### -leader-election-lock-name `` + +Specifies the name of the ConfigMap, within the same namespace as the controller, used as the lock for leader election. + +Requires [-enable-leader-election](#cmdoption-enable-leader-election). + + + +--- + +### -log_backtrace_at `` + +When logging hits line `file:N`, emit a stack trace. + + + +--- + +### -main-template-path `` + +Path to the main NGINX configuration template. + +- Default for NGINX is `nginx.tmpl`. +- Default for NGINX Plus is `nginx-plus.tmpl`. + + + +--- + +### -nginx-configmaps `` + +A ConfigMap resource for customizing NGINX configuration. If a ConfigMap is set, but NGINX Ingress Controller is not able to fetch it from Kubernetes API, NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -mgmt-configmap `` + +The Management ConfigMap resource is used for customizing the NGINX mgmt block. If using NGINX Plus, a Management ConfigMap must be set. If NGINX Ingress Controller is not able to fetch it from Kubernetes API, NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -nginx-debug + +Enable debugging for NGINX. Uses the nginx-debug binary. Requires 'error-log-level: debug' in the ConfigMap. + + + +--- + +### -nginx-plus + +Enable support for NGINX Plus. + + + +--- + +### -nginx-reload-timeout `` + +Timeout in milliseconds which NGINX Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. + +Default is 60000. + + + +--- + +### -nginx-status + +Enable the NGINX stub_status, or the NGINX Plus API. + +Default `true`. + + + +--- + +### -nginx-status-allow-cidrs `` + +Add IP/CIDR blocks to the allow list for NGINX stub_status or the NGINX Plus API. + +Separate multiple IP/CIDR by commas. (default `127.0.0.1,::1`) + + + +--- + +### -nginx-status-port `` + +Set the port where the NGINX stub_status or the NGINX Plus API is exposed. + +Format: `[1024 - 65535]` (default `8080`) + + + +--- + +### -proxy `` + +{{< warning >}} This argument is intended for testing purposes only. {{< /warning >}} + +Use a proxy server to connect to Kubernetes API started with `kubectl proxy`. + +NGINX Ingress Controller does not start NGINX and does not write any generated NGINX configuration files to disk. + + + +--- + +### -report-ingress-status + +Updates the address field in the status of Ingress resources. + +Requires the [-external-service](#cmdoption-external-service) or [-ingresslink](#cmdoption-ingresslink) flag, or the `external-status-address` key in the ConfigMap. + + + +--- + +### -transportserver-template-path `` + +Path to the TransportServer NGINX configuration template for a TransportServer resource. + +- Default for NGINX is `nginx.transportserver.tmpl`. +- Default for NGINX Plus is `nginx-plus.transportserver.tmpl`. + + + +--- + +### -log-level `` + +Log level for Ingress Controller logs. Allowed values: fatal, error, warn, info, debug, trace. + +- Default is `info`. + + + +--- + +### -log-format `` + +Log format for Ingress Controller logs. Allowed values: glog, json, text. + +- Default is `glog`. + + + +--- + +### -version + +Print the version, git-commit hash and build date and exit. + + + +--- + +### -virtualserver-template-path `` + +Path to the VirtualServer NGINX configuration template for a VirtualServer resource. + +- Default for NGINX is `nginx.virtualserver.tmpl`. +- Default for NGINX Plus is `nginx-plus.virtualserver.tmpl`. + + + + +--- + +### -vmodule `` + +A comma-separated list of pattern=N settings for file-filtered logging. + + + +--- + +### -watch-namespace `` + +Comma separated list of namespaces NGINX Ingress Controller should watch for resources. By default NGINX Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace-label". + + + +--- + +### -watch-namespace-label `` + +Configures NGINX Ingress Controller to watch only those namespaces with label foo=bar. By default NGINX Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace". + + + +--- + +### -watch-secret-namespace `` + +Comma separated list of namespaces NGINX Ingress Controller should watch for secrets. If this arg is not configured, NGINX Ingress Controller watches the same namespaces for all resources, see "watch-namespace" and "watch-namespace-label". All namespaces included with this argument must be part of either `-watch-namespace` or `-watch-namespace-label`. + + + +--- + +### -enable-prometheus-metrics + +Enables exposing NGINX or NGINX Plus metrics in the Prometheus format. + + + +--- + +### -prometheus-metrics-listen-port `` + +Sets the port where the Prometheus metrics are exposed. + +Format: `[1024 - 65535]` (default `9113`) + + + +--- + +### -prometheus-tls-secret `` + +A Secret with a TLS certificate and key for TLS termination of the Prometheus metrics endpoint. + +- If the argument is not set, the Prometheus endpoint will not use a TLS connection. +- If the argument is set, but NGINX Ingress Controller is not able to fetch the Secret from Kubernetes API, NGINX Ingress Controller will fail to start. + + + +--- + +### -enable-service-insight + +Exposes the Service Insight endpoint for Ingress Controller. + + + +--- + +### -service-insight-listen-port `` + +Sets the port where the Service Insight is exposed. + +Format: `[1024 - 65535]` (default `9114`) + + + +--- + +### -service-insight-tls-secret `` + +A Secret with a TLS certificate and key for TLS termination of the Service Insight endpoint. + +- If the argument is not set, the Service Insight endpoint will not use a TLS connection. +- If the argument is set, but NGINX Ingress Controller is not able to fetch the Secret from Kubernetes API, NGINX Ingress Controller will fail to start. + +Format: `/` + + + +--- + +### -spire-agent-address `` + +Specifies the address of a running Spire agent. **For use with NGINX Service Mesh only**. + +- If the argument is set, but NGINX Ingress Controller is unable to connect to the Spire Agent, NGINX Ingress Controller will fail to start. + + + + +--- + +### -enable-internal-routes + +Enable support for internal routes with NGINX Service Mesh. **For use with NGINX Service Mesh only**. + +Requires [-spire-agent-address](#cmdoption-spire-agent-address). + +- If the argument is set, but `spire-agent-address` is not provided, NGINX Ingress Controller will fail to start. + + + +--- + +### -enable-latency-metrics + +Enable collection of latency metrics for upstreams. +Requires [-enable-prometheus-metrics](#cmdoption-enable-prometheus-metrics). + + + +--- + +### -enable-app-protect + +Enables support for App Protect. + +Requires [-nginx-plus](#cmdoption-nginx-plus). + +- If the argument is set, but `nginx-plus` is set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-log-level `` + +Sets log level for App Protect. Allowed values: fatal, error, warn, info, debug, trace. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect](#cmdoption-enable-app-protect). + +- If the argument is set, but `nginx-plus` and `enable-app-protect` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -enable-app-protect-dos + +Enables support for App Protect DoS. + +Requires [-nginx-plus](#cmdoption-nginx-plus). + +- If the argument is set, but `nginx-plus` is set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-dos-debug + +Enable debugging for App Protect DoS. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). + +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-dos-max-daemons + +Max number of ADMD instances. + +Default `1`. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). + +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-dos-max-workers + +Max number of nginx processes to support. + +Default `Number of CPU cores in the machine`. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). + +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -app-protect-dos-memory + +RAM memory size to consume in MB + +Default `50% of free RAM in the container or 80MB, the smaller`. + +Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). + +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + + + +--- + +### -ready-status + +Enables the readiness endpoint `/nginx-ready`. The endpoint returns a success code when NGINX has loaded all the config after the startup. + +Default `true`. + + + +--- + +### -ready-status-port + +The HTTP port for the readiness endpoint. + +Format: `[1024 - 65535]` (default `8081`) + +--- + +### -disable-ipv6 + +Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. + +Default `false`. + + + +--- + +### -default-http-listener-port + +Sets the port for the HTTP `default_server` listener. + +Default `80`. + + + +--- + +### -default-https-listener-port + +Sets the port for the HTTPS `default_server` listener. + +Default `443`. + + + +--- + +### -ssl-dynamic-reload + +Used to activate or deactivate lazy loading for SSL Certificates. + +The default value is `true`. + + + +--- + +### -weight-changes-dynamic-reload + +Enables the ability to change the weight distribution of two-way split clients without reloading NGINX. + +Requires [-nginx-plus](#cmdoption-nginx-plus). + +Using this feature may require increasing `map_hash_bucket_size`, `map_hash_max_size`, `variable_hash_bucket_size`, and `variable_hash_max_size` in the ConfigMap based on the number of two-way splits. + +The default value is `false`. + +- If the argument is set, but `nginx-plus` is set to false, NGINX Ingress Controller will ignore the flag. + + + +--- + +### -enable-telemetry-reporting + +Enable gathering and reporting of software telemetry. + +The default value is `true`. + + + +--- + +### -agent + +Enable NGINX Agent which can used with `-enable-app-protect` to send events to Security Monitoring. + +The default value is `false`. + + + +--- + +### -agent-instance-group + +Specify the instance group name to use for the NGINX Ingress Controller deployment when using `-agent`. + + diff --git a/content/nic/configuration/global-configuration/configmap-resource.md b/content/nic/configuration/global-configuration/configmap-resource.md new file mode 100644 index 000000000..711ac7065 --- /dev/null +++ b/content/nic/configuration/global-configuration/configmap-resource.md @@ -0,0 +1,250 @@ +--- +title: ConfigMap resources +weight: 300 +toc: true +type: how-to +product: NIC +docs: DOCS-586 +--- + +When using F5 NGINX Ingress Controller, you can customize or fine tune NGINX behavior using ConfigMap resources. Examples include setting the number of worker processes or customizing the access log format. + +## Using ConfigMap + +1. The [Installation with Manifests]({{< ref "/nic/installation/installing-nic/installation-with-manifests.md" >}}) documentation deploy an empty ConfigMap while the default installation manifests specify it in the command-line arguments of the Ingress Controller. However, if you customized the manifests, to use ConfigMap, make sure to specify the ConfigMap resource to use the [command-line arguments]({{< ref "/nic/configuration/global-configuration/command-line-arguments" >}}) of NGINX Ingress Controller. + +1. Create a ConfigMap file with the name *nginx-config.yaml* and set the values +that make sense for your setup: + + ```yaml + kind: ConfigMap + apiVersion: v1 + metadata: + name: nginx-config + namespace: nginx-ingress + data: + proxy-connect-timeout: "10s" + proxy-read-timeout: "10s" + client-max-body-size: "2m" + ``` + + See the section [Summary of ConfigMap Keys](#summary-of-configmap-keys) for the explanation of the available ConfigMap keys (such as `proxy-connect-timeout` in this example). + +1. Create a new (or update the existing) ConfigMap resource: + + ```shell + kubectl apply -f nginx-config.yaml + ``` + + The NGINX configuration will be updated. + +--- + +## ConfigMap and Ingress annotations + +ConfigMap applies globally, meaning that it affects every Ingress resource. In contrast, annotations always apply to their Ingress resource. Annotations can override some ConfigMap keys: an example is that the `nginx.org/proxy-connect-timeout` annotations overrides the `proxy-connect-timeout` ConfigMap key. + +For more information, view the [Advanced configuration with annotations]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-annotations" >}}) topic. + +--- + +## ConfigMap and VirtualServer/VirtualServerRoute resources + +The ConfigMap affects every VirtualServer and VirtualServerRoute resources. However, the fields of those resources allow overriding some ConfigMap keys. For example, the `connect-timeout` field of the `upstream` overrides the `proxy-connect-timeout` ConfigMap key. + +For more information, view the [VirtualServer and VirtualServerRoute resources]({{< ref "/nic/configuration/virtualserver-and-virtualserverroute-resources" >}}) topic. + +--- + +## ConfigMap keys + +### Ingress Controller (Unrelated to NGINX Configuration) + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*external-status-address* | Sets the address to be reported in the status of Ingress resources. Requires the *-report-status* command-line argument. Overrides the *-external-service* argument. | N/A | [Reporting resource status]({{< ref "/nic/configuration/global-configuration/reporting-resources-status" >}}) | +{{}} + +--- + +### General customization + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*proxy-connect-timeout* | Sets the value of the [proxy_connect_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) and [grpc_connect_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout) directive. | *60s* | | +|*proxy-read-timeout* | Sets the value of the [proxy_read_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) and [grpc_read_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout) directive. | *60s* | | +|*proxy-send-timeout* | Sets the value of the [proxy_send_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout) and [grpc_send_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_send_timeout) directive. | *60s* | | +|*client-max-body-size* | Sets the value of the [client_max_body_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) directive. | *1m* | | +|*proxy-buffering* | Enables or disables [buffering of responses](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) from the proxied server. | *True* | | +|*proxy-buffers* | Sets the value of the [proxy_buffers](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directive. | Depends on the platform. | | +|*proxy-buffer-size* | Sets the value of the [proxy_buffer_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) and [grpc_buffer_size](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_buffer_size) directives. | Depends on the platform. | | +|*proxy-max-temp-file-size* | Sets the value of the [proxy_max_temp_file_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size) directive. | *1024m* | | +|*set-real-ip-from* | Sets the value of the [set_real_ip_from](https://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from) directive. | N/A | | +|*real-ip-header* | Sets the value of the [real_ip_header](https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header) directive. | *X-Real-IP* | | +|*real-ip-recursive* | Enables or disables the [real_ip_recursive](https://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_recursive) directive. | *False* | | +|*default-server-return* | Configures the [return](https://nginx.org/en/docs/http/ngx_http_rewrite_module.html#return) directive in the default server, which handles a client request if none of the hosts of Ingress or VirtualServer resources match. The default value configures NGINX to return a 404 error page. You can configure a fixed response or a redirect. For example, *default-server-return: 302 https://nginx.org* will redirect a client to *https://nginx.org*. | *404* | | +|*server-tokens* | Enables or disables the [server_tokens](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens) directive. Additionally, with the NGINX Plus, you can specify a custom string value, including the empty string value, which disables the emission of the “Server” field. | *True* | | +|*worker-processes* | Sets the value of the [worker_processes](https://nginx.org/en/docs/ngx_core_module.html#worker_processes) directive. | *auto* | | +|*worker-rlimit-nofile* | Sets the value of the [worker_rlimit_nofile](https://nginx.org/en/docs/ngx_core_module.html#worker_rlimit_nofile) directive. | N/A | | +|*worker-connections* | Sets the value of the [worker_connections](https://nginx.org/en/docs/ngx_core_module.html#worker_connections) directive. | *1024* | | +|*worker-cpu-affinity* | Sets the value of the [worker_cpu_affinity](https://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity) directive. | N/A | | +|*worker-shutdown-timeout* | Sets the value of the [worker_shutdown_timeout](https://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout) directive. | N/A | | +|*server-names-hash-bucket-size* | Sets the value of the [server_names_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size) directive. | *256* | | +|*server-names-hash-max-size* | Sets the value of the [server_names_hash_max_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size) directive. | *1024* | | +|*map-hash-bucket-size* | Sets the value of the [map_hash_bucket_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_bucket_size) directive.| *256* | | +|*map-hash-max-size* | Sets the value of the [map_hash_max_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_max_size) directive. | *2048* | | +|*resolver-addresses* | Sets the value of the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) addresses. Note: If you use a DNS name (for example, *kube-dns.kube-system.svc.cluster.local* ) as a resolver address, NGINX Plus will resolve it using the system resolver during the start and on every configuration reload. If the name cannot be resolved or the DNS server doesn't respond, NGINX Plus will fail to start or reload. To avoid this, we recommend using IP addresses as resolver addresses instead of DNS names. Supported in NGINX Plus only. | N/A | [Support for Type ExternalName Services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services). | +|*resolver-ipv6* | Enables IPv6 resolution in the resolver. Supported in NGINX Plus only. | *True* | [Support for Type ExternalName Services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services). | +|*resolver-valid* | Sets the time NGINX caches the resolved DNS records. Supported in NGINX Plus only. | TTL value of a DNS record | [Support for Type ExternalName Services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services). | +|*resolver-timeout* | Sets the [resolver_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout) for name resolution. Supported in NGINX Plus only. | *30s* | [Support for Type ExternalName Services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services). | +|*keepalive-timeout* | Sets the value of the [keepalive_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) directive. | *75s* | | +|*keepalive-requests* | Sets the value of the [keepalive_requests](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests) directive. | *1000* | | +|*variables-hash-bucket-size* | Sets the value of the [variables_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables_hash_bucket_size) directive. | *256* | | +|*variables-hash-max-size* | Sets the value of the [variables-hash-max-size](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables_hash_max_size) directive. | *1024* | | +{{}} + +--- + +### Logging + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*error-log-level* | Sets the global [error log level](https://nginx.org/en/docs/ngx_core_module.html#error_log) for NGINX. | *notice* | | +|*access-log* | Sets the directive [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log). A syslog destination is the only valid value. The value will be set to its default in-case user tries to configure it with location other than a syslog. +| ``/dev/stdout main`` | ``syslog:server=localhost:514`` | +|*access-log-off* | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log). | *False* | | +|*default-server-access-log-off* | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log) for the default server. If access log is disabled globally (*access-log-off: "True"*), then the default server access log is always disabled. | *False* | | +|*log-format* | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for HTTP and HTTPS traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by *\n*). In that case, the Ingress Controller will replace every *\n* character with a space character. All *'* characters must be escaped. | See the [template file](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/internal/configs/version1/nginx.tmpl) for the access log. | [Custom Log Format](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/shared-examples/custom-log-format). | +|*log-format-escaping* | Sets the characters escaping for the variables of the log format. Supported values: *json* (JSON escaping), *default* (the default escaping) *none* (disables escaping). | *default* | | +|*stream-log-format* | Sets the custom [log format](https://nginx.org/en/docs/stream/ngx_stream_log_module.html#log_format) for TCP, UDP, and TLS Passthrough traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by *\n*). In that case, the Ingress Controller will replace every *\n* character with a space character. All *'* characters must be escaped. | See the [template file](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/internal/configs/version1/nginx.tmpl). | | +|*stream-log-format-escaping* | Sets the characters escaping for the variables of the stream log format. Supported values: *json* (JSON escaping), *default* (the default escaping) *none* (disables escaping). | *default* | | +{{}} + +--- + +### Request URI/Header manipulation + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*proxy-hide-headers* | Sets the value of one or more [proxy_hide_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directives. Example: *"nginx.org/proxy-hide-headers": "header-a,header-b"* | N/A | | +|*proxy-pass-headers* | Sets the value of one or more [proxy_pass_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directives. Example: *"nginx.org/proxy-pass-headers": "header-a,header-b"* | N/A | | +{{}} + +--- + +### Auth and SSL/TLS + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*redirect-to-https* | Sets the 301 redirect rule based on the value of the *http_x_forwarded_proto* header on the server block to force incoming traffic to be over HTTPS. Useful when terminating SSL in a load balancer in front of the Ingress Controller — see [115](https://github.com/nginx/kubernetes-ingress/issues/115) | *False* | | +|*ssl-redirect* | Sets an unconditional 301 redirect rule for all incoming HTTP traffic to force incoming traffic over HTTPS. | *True* | | +|*hsts* | Enables [HTTP Strict Transport Security (HSTS)](https://www.nginx.com/blog/http-strict-transport-security-hsts-and-nginx/) : the HSTS header is added to the responses from backends. The *preload* directive is included in the header. | *False* | | +|*hsts-max-age* | Sets the value of the *max-age* directive of the HSTS header. | *2592000* (1 month) | | +|*hsts-include-subdomains* | Adds the *includeSubDomains* directive to the HSTS header. | *False* | | +|*hsts-behind-proxy* | Enables HSTS based on the value of the *http_x_forwarded_proto* request header. Should only be used when TLS termination is configured in a load balancer (proxy) in front of the Ingress Controller. Note: to control redirection from HTTP to HTTPS configure the *nginx.org/redirect-to-https* annotation. | *False* | | +|*ssl-protocols* | Sets the value of the [ssl_protocols](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_protocols) directive. | *TLSv1 TLSv1.1 TLSv1.2* | | +|*ssl-prefer-server-ciphers* | Enables or disables the [ssl_prefer_server_ciphers](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_prefer_server_ciphers) directive. | *False* | | +|*ssl-ciphers* | Sets the value of the [ssl_ciphers](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers) directive. | *HIGH:!aNULL:!MD5* | | +|*ssl-dhparam-file* | Sets the content of the dhparam file. The controller will create the file and set the value of the [ssl_dhparam](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam) directive with the path of the file. | N/A | | +{{}} + +--- + +### Listeners + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*http2* | Enables HTTP/2 in servers with SSL enabled. | *False* | | +|*proxy-protocol* | Enables PROXY Protocol for incoming connections. | *False* | [Proxy Protocol](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/shared-examples/proxy-protocol). | +{{}} + +--- + +### Backend services (Upstreams) + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*lb-method* | Sets the [load balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify *"round_robin"*. | *"random two least_conn"* | | +|*max-fails* | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the *server* directive. | *1* | | +|*upstream-zone-size* | Sets the size of the shared memory [zone](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#zone) for upstreams. For NGINX, the special value 0 disables the shared memory zones. For NGINX Plus, shared memory zones are required and cannot be disabled. The special value 0 will be ignored. | *256k* for NGINX, *512k* for NGINX Plus | | +|*fail-timeout* | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the *server* directive. | *10s* | | +|*keepalive* | Sets the value of the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. Note that *proxy_set_header Connection "";* is added to the generated configuration when the value > 0. | *0* | | +{{}} + +--- + +### Zone Sync + +Zone Sync enables the [ngx_stream_zone_sync_module](https://nginx.org/en/docs/stream/ngx_stream_zone_sync_module.html) in NGINX Ingress Controller when NGINX Plus is used. Multiple replicas are required to effectively utililise this functionality. More information is available in the [How NGINX Plus Performs Zone Synchronization](https://docs.nginx.com/nginx/admin-guide/high-availability/zone_sync_details/) topic. + +Zone synchronization with TLS for NGINX Ingress Controller is not yet available with ConfigMap. If you would like to enable Zone Sync with TLS, please remove `zone-sync` from ConfigMap and add Zone Sync parameters via [`stream-snippets`]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md" >}}) similar to [this example](https://github.com/nginx/kubernetes-ingress/blob/v4.0.1/examples/custom-resources/oidc/nginx-config.yaml) and adding the [zone_sync_ssl directive](https://nginx.org/en/docs/stream/ngx_stream_zone_sync_module.html#zone_sync_ssl) along with any other TLS parameters to the `stream-snippets`. + +You will also need to manually add the headless service, such as in [this example](https://github.com/nginx/kubernetes-ingress/blob/v4.0.1/examples/custom-resources/oidc/nginx-ingress-headless.yaml). + +{{< caution >}} +If you previously installed OIDC or used the `zone_sync` directive with `stream-snippets` in [v4.0.1](https://github.com/nginx/kubernetes-ingress/tree/v4.0.1) or earlier, and you plan to enable the `zone-sync` ConfigMap key, the `zone_sync` directive should be removed from `stream-snippets`. + +If you encounter the error `error [emerg] 13#13: "zone_sync" directive is duplicate in /etc/nginx/nginx.conf:164` it is likely due to `zone_sync` being enabled in both `stream-snippets` and the ConfigMap. Once upgraded, remove the [old headless service](https://github.com/nginx/kubernetes-ingress/blob/v4.0.1/examples/custom-resources/oidc/nginx-ingress-headless.yaml) deployed for OIDC. +{{< /caution >}} + + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*zone-sync* | Enables zone synchronization between NGINX Ingress Controller Pods. This autogenerates a [zone_sync_server](https://nginx.org/en/docs/stream/ngx_stream_zone_sync_module.html#zone_sync_server) and a headless service using the `ReplicaSet` or `DaemonSet` name. Please note that this headless service will be automatically cleaned up when uninstalling via Helm or by removing the value from the ConfigMap. The headless service will need to be manually removed if the `controller.customConfigMap` value is set via Helm or the deployment is uninstalled via Manifests. Each Ingress Controller manages its own headless service. NGINX Plus Required. | *False* | | +|*zone-sync-port* | Specifies the optional port on which NGINX Ingress Controller listens for zone sync traffic. NGINX Plus & `zone-sync` Required. | *12345* | | +|*zone-sync-resolver-addresses* | Configures optional addresses used in the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) directive for zone-sync. This field takes a comma separated list of addresses. NGINX Plus & `zone-sync` Required | `kube-dns.kube-system.svc.cluster.local` | | +|*zone-sync-resolver-ipv6* | Configures whether the optional [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) directive for zone-sync will look up IPv6 addresses. NGINX Plus & `zone-sync` Required | `true` | | +|*zone-sync-resolver-valid* | Configures an [NGINX time](https://nginx.org/en/docs/syntax.html) that the optional [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) directive for zone-sync will override the TTL value of responses from nameservers with. NGINX Plus & `zone-sync` Required | `5s` | | +{{}} + +--- + +### Snippets and custom templates + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*main-snippets* | Sets a custom snippet in main context. | N/A | | +|*http-snippets* | Sets a custom snippet in http context. | N/A | | +|*location-snippets* | Sets a custom snippet in location context. | N/A | | +|*server-snippets* | Sets a custom snippet in server context. | N/A | | +|*stream-snippets* | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/tcp-udp). | +|*main-template* | Sets the main NGINX configuration template. | By default the template is read from the file in the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | +|*ingress-template* | Sets the NGINX configuration template for an Ingress resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | +|*virtualserver-template* | Sets the NGINX configuration template for an VirtualServer resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | +|*transportserver-template* | Sets the NGINX configuration template for a TransportServer resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | +{{}} + +--- + +### Modules + +{{}} +|ConfigMap Key | Description | Default | Example | +| ---| ---| ---| --- | +|*otel-exporter-endpoint* | OTLP/gRPC endpoint that will accept [OpenTelemetry](https://opentelemetry.io) data. Set `otel-trace-in-http` to *"true"* to enable OpenTelemetry at the global level. | N/A | *"https://otel-collector:4317"* | +|*otel-exporter-header-name* | The name of a custom HTTP header to add to telemetry export request. `otel-exporter-endpoint` and `otel-exporter-header-value` required. | N/A | *"X-custom-header"* | +|*otel-exporter-header-value* | The value of a custom HTTP header to add to telemetry export request. `otel-exporter-endpoint` and `otel-exporter-header-name` required. | N/A | *"custom-value"* | +|*otel-service-name* | Sets the `service.name` attribute of the OTel resource. `otel-exporter-endpoint` required. | N/A | *"nginx-ingress-controller:nginx"* | +| *otel-trace-in-http* | Enables [OpenTelemetry](https://opentelemetry.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Set this to *"false"* to enable OpenTelemetry for individual routes with snippets. `otel-exporter-endpoint` required. | *"false"* | *"true"* | +|*opentracing* | Removed in v5.0.0. Enables [OpenTracing](https://opentracing.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Note: requires the Ingress Controller image with OpenTracing module and a tracer. See the [docs]({{< ref "/nic/installation/integrations/opentracing.md" >}}) for more information. | *False* | | +|*opentracing-tracer* | Removed in v5.0.0. Sets the path to the vendor tracer binary plugin. | N/A | | +|*opentracing-tracer-config* | Removed in v5.0.0. Sets the tracer configuration in JSON format. | N/A | | +|*app-protect-compressed-requests-action* | Sets the *app_protect_compressed_requests_action* [global directive](/nginx-app-protect/configuration/#global-directives). | *drop* | | +|*app-protect-cookie-seed* | Sets the *app_protect_cookie_seed* [global directive](/nginx-app-protect/configuration/#global-directives). | Random automatically generated string | | +|*app-protect-failure-mode-action* | Sets the *app_protect_failure_mode_action* [global directive](/nginx-app-protect/configuration/#global-directives). | *pass* | | +|*app-protect-cpu-thresholds* | Sets the *app_protect_cpu_thresholds* [global directive](/nginx-app-protect/configuration/#global-directives). | *high=100 low=100* | | +|*app-protect-physical-memory-util-thresholds* | Sets the *app_protect_physical_memory_util_thresholds* [global directive](/nginx-app-protect/configuration/#global-directives). | *high=100 low=100* | | +|`app-protect-reconnect-period-seconds` | Sets the `app_protect_reconnect_period_seconds` [global directive](/nginx-app-protect/configuration/#global-directives). | `5` | | +|*app-protect-dos-log-format* | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for Dos Access log traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by *\n*). In that case, the Ingress Controller will replace every *\n* character with a space character. All *'* characters must be escaped. | `, vs_name_al=$app_protect_dos_vs_name, ip=$remote_addr, tls_fp=$app_protect_dos_tls_fp, outcome=$app_protect_dos_outcome, reason=$app_protect_dos_outcome_reason, policy_name=$app_protect_dos_policy_name, dos_version=$app_protect_dos_version, ip_tls=$remote_addr:$app_protect_dos_tls_fp,` | | +|*app-protect-dos-log-format-escaping* | Sets the characters escaping for the variables of the stream log format. Supported values: *json* (JSON escaping), *default* (the default escaping) *none* (disables escaping). | *default* | | +|*app-protect-dos-arb-fqdn* | Sets the *app-protect-dos-arb-fqdn* [directive](/nginx-app-protect-dos/directives-and-policy/learn-about-directives-and-policy/#arbitrator-fqdn-directive-app_protect_dos_arb_fqdn). | *svc-appprotect-dos-arb* | | +{{}} diff --git a/content/nic/configuration/global-configuration/custom-templates.md b/content/nic/configuration/global-configuration/custom-templates.md new file mode 100644 index 000000000..589ce5951 --- /dev/null +++ b/content/nic/configuration/global-configuration/custom-templates.md @@ -0,0 +1,11 @@ +--- +docs: DOCS-587 +doctypes: +- '' +title: Custom templates +toc: true +weight: 500 +--- + + +F5 NGINX Ingress Controller uses templates to generate NGINX configuration for Ingress resources, VirtualServer resources and the main NGINX configuration file. You can customize the templates and apply them via the ConfigMap. See the [corresponding example](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/shared-examples/custom-templates). diff --git a/content/nic/configuration/global-configuration/globalconfiguration-resource.md b/content/nic/configuration/global-configuration/globalconfiguration-resource.md new file mode 100644 index 000000000..84b4e4002 --- /dev/null +++ b/content/nic/configuration/global-configuration/globalconfiguration-resource.md @@ -0,0 +1,183 @@ +--- +docs: DOCS-588 +doctypes: +- '' +title: GlobalConfiguration resource +toc: true +weight: 200 +--- + +This page explains how to use the GlobalConfiguration resource to define the global configuration parameters of F5 NGINX Ingress Controller. + +The resource supports configuring listeners for TCP and UDP load balancing, and is implemented as a [Custom resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + +Listeners are required by [TransportServer resources]({{< ref "/nic/configuration/transportserver-resource.md" >}}) and can be used to [configure custom listeners for VirtualServers]({{< ref "/nic/tutorials/virtual-server-with-custom-listener-ports.md" >}}). + +--- + +## Prerequisites + +When [installing NGINX Ingress Controller using Manifests]({{< ref "/nic/installation/installing-nic/installation-with-manifests.md" >}}), you need to reference a GlobalConfiguration resource in the [`-global-configuration`](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments#cmdoption-global-configuration) command-line argument. NGINX Ingress Controller only needs one GlobalConfiguration resource. + +--- + +## GlobalConfiguration specification + +The GlobalConfiguration resource defines the global configuration parameters of the Ingress Controller. Below is an example: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: GlobalConfiguration +metadata: + name: nginx-configuration + namespace: nginx-ingress +spec: + listeners: + - name: dns-udp + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP + - name: http-8083 + port: 8083 + protocol: HTTP + - name: https-8443 + port: 8443 + protocol: HTTP + ssl: true +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +| *listeners* | A list of listeners. | [listener](#listener) | No | +{{}} + +### Listener + +The `listeners:` key defines a listener (a combination of a protocol and a port) that NGINX will use to accept traffic for a [TransportServer]({{< ref "/nic/configuration/transportserver-resource.md" >}}) and a [VirtualServer]({{< ref "/nic/configuration/virtualserver-and-virtualserverroute-resources.md" >}}): + +```yaml +- name: dns-tcp + port: 5353 + protocol: TCP +- name: http-8083 + port: 8083 + protocol: HTTP +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +| *name* | The name of the listener. Must be a valid DNS label as defined in RFC 1035. For example, ``hello`` and ``listener-123`` are valid. The name must be unique among all listeners. The name ``tls-passthrough`` is reserved for the built-in TLS Passthrough listener and cannot be used. | *string* | Yes | +| *port* | The port of the listener. The port must fall into the range ``1..65535`` with the following exceptions: ``80``, ``443``, the [status port](/nginx-ingress-controller/logging-and-monitoring/status-page), the [Prometheus metrics port](/nginx-ingress-controller/logging-and-monitoring/prometheus). Among all listeners, only a single combination of a port-protocol is allowed. | *int* | Yes | +| *protocol* | The protocol of the listener. Supported values: ``TCP``, ``UDP`` and ``HTTP``. | *string* | Yes | +| *ssl* | Configures the listener with SSL. This is currently only supported for ``HTTP`` listeners. Default value is ``false`` | *bool* | No | +| *ipv4* | Specifies the IPv4 address to listen on. | *string* | No | +| *ipv6* | Specifies the IPv6 address to listen on. | *string* | No | + +{{}} + +--- + +## Using GlobalConfiguration + +You can use the usual `kubectl` commands to work with a GlobalConfiguration resource. + +For example, the following command creates a GlobalConfiguration resource defined in `global-configuration.yaml` with the name `nginx-configuration`: + +```shell +kubectl apply -f global-configuration.yaml +``` +```shell +globalconfiguration.k8s.nginx.org/nginx-configuration created +``` + +Assuming the namespace of the resource is `nginx-ingress`, you can get the resource by running: + +```shell +kubectl get globalconfiguration nginx-configuration -n nginx-ingress +``` +```shell +NAME AGE +nginx-configuration 13s +``` + +With `kubectl get` and similar commands, you can use the short name `gc` instead of `globalconfiguration`. + +--- + +### Validation + +Two types of validation are available for the GlobalConfiguration resource: + +- *Structural validation* by `kubectl` and Kubernetes API server. +- *Comprehensive validation* by NGINX Ingress Controller. + + +#### Structural validation + +The custom resource definition for the GlobalConfiguration includes structural OpenAPI schema which describes the type of every field of the resource. + +If you try to create (or update) a resource that violates the structural schema (for example, you use a string value for the port field of a listener), `kubectl` and Kubernetes API server will reject such a resource: + +- Example of `kubectl` validation: + + ```shell + kubectl apply -f global-configuration.yaml + ``` + ```text + error: error validating "global-configuration.yaml": error validating data: ValidationError(GlobalConfiguration.spec.listeners[0].port): invalid type for org.nginx.k8s.v1.GlobalConfiguration.spec.listeners.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + ``` + +- Example of Kubernetes API server validation: + + ```shell + kubectl apply -f global-configuration.yaml --validate=false + ``` + ```text + The GlobalConfiguration "nginx-configuration" is invalid: []: Invalid value: map[string]interface {}{ ... }: validation failure list: + spec.listeners.port in body must be of type integer: "string" + ``` + +If a resource is not rejected (it doesn't violate the structural schema), NGINX Ingress Controller will validate it further. + +#### Comprehensive validation + +NGINX Ingress Controller validates the fields of a GlobalConfiguration resource. If a GlobalConfiguration resource is partially invalid, NGINX Ingress Controller use the valid listeners and emit events about invalid listeners. + +You can check if the Ingress Controller successfully applied the configuration for a GlobalConfiguration. For our `nginx-configuration` GlobalConfiguration, we can run: + +```shell +kubectl describe gc nginx-configuration -n nginx-ingress +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Updated 11s nginx-ingress-controller GlobalConfiguration nginx-ingress/nginx-configuration was updated +``` + +The events section includes a Normal event with the Updated reason that informs us that the configuration was successfully applied. + +If you create a GlobalConfiguration `nginx-configuration` with two or more listeners that have the same protocol UDP and port 53, you will get: + +```shell +kubectl describe gc nginx-configuration -n nginx-ingress +``` +```text +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Updated 55s nginx-ingress-controller GlobalConfiguration nginx-ingress/nginx-configuration was updated + Warning AddedOrUpdatedWithError 6s nginx-ingress-controller GlobalConfiguration nginx-ingress/nginx-configuration is invalid and was rejected: spec.listeners: Duplicate value: "Duplicated port/protocol combination 53/UDP" +``` + +The events section includes a Warning event with the AddedOrUpdatedWithError reason. + + +## Using IPV4 and IPV6 Addresses with GlobalConfiguration + +You can customize the IPv4 and IPv6 Address listeners in the global configuration and apply them to your VirtualServer resources. See the corresponding example [here](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/custom-ip-listeners/virtualserver/) diff --git a/content/nic/configuration/global-configuration/mgmt-configmap-resource.md b/content/nic/configuration/global-configuration/mgmt-configmap-resource.md new file mode 100644 index 000000000..e1d9a120c --- /dev/null +++ b/content/nic/configuration/global-configuration/mgmt-configmap-resource.md @@ -0,0 +1,50 @@ +--- +docs: DOCS-586 +doctypes: +- '' +title: Management ConfigMap resource +toc: true +weight: 300 +--- + +When using F5 NGINX Ingress Controller with NGINX Plus, it is required to pass a [command line argument]({{< ref "/nic/configuration/global-configuration/command-line-arguments" >}}) to NGINX Ingress Controller, `--mgmt-configmap=` which specifies the ConfigMap to use. The minimal required ConfigMap must have a `license-token-secret-name` key. Helm users will not need to create this map or pass the argument, it will be created with a Helm install. + +--- + +1. Create a ConfigMap file with the name *nginx-config-mgmt.yaml* and set the values +that make sense for your setup: + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: nginx-config-mgmt + namespace: nginx-ingress + data: + license-token-secret-name: "license-token" + ``` +1. Create a new (or update the existing) ConfigMap resource: + + ```shell + kubectl apply -f nginx-config-mgmt.yaml + ``` + + The [NGINX Management](https://nginx.org/en/docs/ngx_mgmt_module.html) block configuration will be updated. +--- +## Management ConfigMap keys + +{{}} +|ConfigMap Key | Description | Default | +| ---| ---| ---| +|*license-token-secret-name* | Configures the secret used in the [license_token](https://nginx.org/en/docs/ngx_mgmt_module.html#license_token) directive. This key assumes the secret is in the Namespace that NGINX Ingress Controller is deployed in. The secret must be of type `nginx.com/license` with the base64 encoded JWT in the `license.jwt` key. | N/A | +|*ssl-verify* | Configures the [ssl_verify](https://nginx.org/en/docs/ngx_mgmt_module.html#ssl_verify) directive, which enables or disables verification of the usage reporting endpoint certificate. | `true` | +|*enforce-initial-report* | Configures the [enforce_initial_report](https://nginx.org/en/docs/ngx_mgmt_module.html#enforce_initial_report) directive, which enables or disables the 180-day grace period for sending the initial usage report. | `false` | +|*usage-report-endpoint* | Configures the endpoint of the [usage_report](https://nginx.org/en/docs/ngx_mgmt_module.html#usage_report) directive. This is used to configure the endpoint NGINX uses to send usage reports to NIM. | `product.connect.nginx.com` | +|*usage-report-interval* | Configures the interval of the [usage_report](https://nginx.org/en/docs/ngx_mgmt_module.html#usage_report) directive. This specifies the frequency that usage reports are sent. This field takes an [NGINX time](https://nginx.org/en/docs/syntax.html). | `1h` | +|*usage-report-proxy-host* | Configures the host name of the [proxy](https://nginx.org/en/docs/ngx_mgmt_module.html#proxy) directive with optional port. | N/A | +|*ssl-trusted-certificate-secret-name* | Configures the secret used to create the file(s) referenced the in [ssl_trusted_certifcate](https://nginx.org/en/docs/ngx_mgmt_module.html#ssl_trusted_certificate), and [ssl_crl](https://nginx.org/en/docs/ngx_mgmt_module.html#ssl_crl) directives. This key assumes the secret is in the Namespace that NGINX Ingress Controller is deployed in. The secret must be of type `nginx.org/ca`, where the `ca.crt` key contains a base64 encoded trusted cert, and the optional `ca.crl` key can contain a base64 encoded CRL. If the optional `ca.crl` key is supplied, it will configure the NGINX `ssl_crl` directive. | N/A | +|*ssl-certificate-secret-name* | Configures the secret used to create the `ssl_certificate` and `ssl_certificate_key` directives. This key assumes the secret is in the Namespace that NGINX Ingress Controller is deployed in. The secret must be of type `kubernetes.io/tls`| N/A | +|*resolver-addresses* | Configures addresses used in the mgmt block [resolver](https://nginx.org/en/docs/ngx_mgmt_module.html#resolver) directive. This field takes a comma separated list of addresses. | N/A | +|*resolver-ipv6* | Configures whether the mgmt block [resolver](https://nginx.org/en/docs/ngx_mgmt_module.html#resolver) directive will look up IPv6 addresses. | `true` | +|*resolver-valid* | Configures an [NGINX time](https://nginx.org/en/docs/syntax.html) that the mgmt block [resolver](https://nginx.org/en/docs/ngx_mgmt_module.html#resolver) directive will override the TTL value of responses from nameservers with. | N/A | +{{}} diff --git a/content/nic/configuration/global-configuration/reporting-resources-status.md b/content/nic/configuration/global-configuration/reporting-resources-status.md new file mode 100644 index 000000000..8e36733eb --- /dev/null +++ b/content/nic/configuration/global-configuration/reporting-resources-status.md @@ -0,0 +1,195 @@ +--- +docs: DOCS-589 +doctypes: +- '' +title: Reporting resource status +toc: true +weight: 600 +--- + +This page describes how to view the status of resources managed by F5 NGINX Ingress Controller. + +## Ingress resources + +An Ingress resource status includes the address (an IP address or a DNS name), through which the hosts of that Ingress resource are publicly accessible. + +You can see the address in the output of the `kubectl get ingress` command, in the ADDRESS column, as shown below: + +```shell +kubectl get ingresses +``` +```text +NAME HOSTS ADDRESS PORTS AGE +cafe-ingress cafe.example.com 12.13.23.123 80, 443 2m +``` + +NGINX Ingress Controller must be configured to report an Ingress status: + +1. Use the command-line flag `-report-ingress-status`. +1. Define a source for an external address. This can be either of: + 1. A user defined address, specified in the `external-status-address` ConfigMap key. + 1. A Service of the type LoadBalancer configured with an external IP or address and specified by the `-external-service` command-line flag. + +View the [ConfigMap keys](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) and [Command-line arguments](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments) topics for more information. + +{{< note >}} NGINX Ingress Controller does not clear the status of Ingress resources when it is being shut down. {{< /note >}} + +## VirtualServer and VirtualServerRoute resources + +A VirtualServer or VirtualServerRoute resource includes the status field with information about the state of the resource and the IP address, through which the hosts of that resource are publicly accessible. + +You can see the status in the output of the `kubectl get virtualservers` or `kubectl get virtualserverroutes` commands as shown below: + +```shell +kubectl get virtualservers +``` +```text + NAME STATE HOST IP PORTS AGE + cafe Valid cafe.example.com 12.13.23.123 [80,443] 34s +``` + +To see an external hostname address associated with a VirtualServer resource, use the `-o wide` option: + +```shell +kubectl get virtualservers -o wide +``` +```text + NAME STATE HOST IP EXTERNALHOSTNAME PORTS AGE + cafe Valid cafe.example.com ae430f41a1a0042908655abcdefghijkl-12345678.eu-west-2.elb.amazonaws.com [80,443] 106s +``` + +{{< note >}} If there are multiple addresses, only the first one is shown. {{< /note >}} + +In order to see additional addresses or extra information about the `Status` of the resource, use the following command: + +```shell +kubectl describe virtualserver +``` +```text +... +Status: + External Endpoints: + Ip: 12.13.23.123 + Ports: [80,443] + Message: Configuration for cafe/cafe was added or updated + Reason: AddedOrUpdated + State: Valid +``` + +### Status specification + +The following fields are reported in both VirtualServer and VirtualServerRoute status: + +{{}} +|Field | Description | Type | +| ---| ---| --- | +|*State* | Current state of the resource. Can be ``Valid``, ``Warning`` an ``Invalid``. For more information, refer to the ``message`` field. | *string* | +|*Reason* | The reason of the last update. | *string* | +|*Message* | Additional information about the state. | *string* | +|*ExternalEndpoints* | A list of external endpoints for which the hosts of the resource are publicly accessible. | *[externalEndpoint](#externalendpoint)* | +{{}} + +The *ReferencedBy* field is reported for the VirtualServerRoute status only: + +{{}} +|Field | Description | Type | +| ---| ---| --- | +| *ReferencedBy* | The VirtualServer that references this VirtualServerRoute. Format as ``namespace/name`` | *string* | +{{}} + +### externalEndpoint + +{{}} +|Field | Description | Type | +| ---| ---| --- | +|``IP`` | The external IP address. | ``string`` | +|``Hostname`` | The external LoadBalancer Hostname address. | ``string`` | +|``Ports`` | A list of external ports. | ``string`` | +{{}} + +NGINX Ingress Controller must be configured to report a VirtualServer or VirtualServerRoute status: + +1. If you want NGINX Ingress Controller to report the `externalEndpoints`, define a source for an external address (The rest of the fields will be reported without the external address configured). This can be: + 1. A user defined address, specified in the `external-status-address` ConfigMap key. + 1. A Service of the type LoadBalancer configured with an external IP or address and specified by the `-external-service` command-line flag. + +View the [ConfigMap keys](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) and [Command-line arguments](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments) topics for more information. + +{{< note >}} NGINX Ingress Controller does not clear the status of VirtualServer and VirtualServerRoute resources when it is being shut down. {{< /note >}} + +## Policy resources + +A Policy resource includes the status field with information about the state of the resource. + +You can see the status in the output of the `kubectl get policy` command as shown below: + +```shell +kubectl get policy +``` +```text + NAME STATE AGE + webapp-policy Valid 30s +``` + +In order to see additional addresses or extra information about the `Status` of the resource, use the following command: + +```shell +kubectl describe policy +``` +```text +... +Status: + Message: Configuration for default/webapp-policy was added or updated + Reason: AddedOrUpdated + State: Valid +``` + +### Status specification + +The following fields are reported in Policy status: + +{{}} +|Field | Description | Type | +| ---| ---| --- | +|``State`` | Current state of the resource. Can be ``Valid`` or ``Invalid``. For more information, refer to the ``message`` field. | ``string`` | +|``Reason`` | The reason of the last update. | ``string`` | +|``Message`` | Additional information about the state. | ``string`` | +{{}} + +## TransportServer resources + +A TransportServer resource includes the status field with information about the state of the resource. + +You can see the status in the output of the `kubectl get transportserver` command as shown below: + +```shell +kubectl get transportserver +``` +```text + NAME STATE REASON AGE + dns-tcp Valid AddedOrUpdated 47m +``` + +To see additional addresses or extra information about the `Status` of the resource, use the following command: + +```shell +kubectl describe transportserver +``` +```text +Status: + Message: Configuration for default/dns-tcp was added or updated + Reason: AddedOrUpdated + State: Valid +``` + +### Status specification + +The following fields are reported in TransportServer status: + +{{}} +|Field | Description | Type | +| ---| ---| --- | +| *State* | Current state of the resource. Can be ``Valid``, ``Warning`` or ``Invalid``. For more information, refer to the ``message`` field. | *string* | +| *Reason* | The reason of the last update. | *string* | +| *Message* | Additional information about the state. | *string* | +{{}} diff --git a/content/nic/configuration/host-and-listener-collisions.md b/content/nic/configuration/host-and-listener-collisions.md new file mode 100644 index 000000000..502ef830c --- /dev/null +++ b/content/nic/configuration/host-and-listener-collisions.md @@ -0,0 +1,168 @@ +--- +title: Host and Listener collisions +toc: true +weight: 800 +docs: DOCS-590 +--- + +This document explains how F5 NGINX Ingress Controller handles host and listener collisions between resources. + +--- + +## Winner Selection Algorithm + +If multiple resources contend for the same host or listener, NGINX Ingress Controller will pick the winner based on the `creationTimestamp` of the resources: the oldest resource will win. In case there are more than one oldest resource (their `creationTimestamp` is the same), NGINX Ingress Controller will choose the resource with the lexicographically smallest `uid`. + +{{< note >}} The `creationTimestamp` and `uid` fields are part of the [ObjectMeta](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/object-meta/) resource. {{< /note >}} + +--- + +## Host collisions + +A host collision occurs when multiple Ingress, VirtualServer, and TransportServer (configured for TLS Passthrough) resources configure the same `host`. NGINX Ingress Controller has two strategies for handling host collisions: + +- Choosing a single "winner" resource to handle the host. +- Merging the configuration of the conflicting resources. + +--- + +### Choosing the winner + +Consider the following two resources: + +- `cafe-ingress` Ingress: + + ```yaml + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cafe-ingress + spec: + ingressClassName: nginx + rules: + - host: cafe.example.com + . . . + ``` + +- `cafe-virtual-server` VirtualServer: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: VirtualServer + metadata: + name: cafe-virtual-server + spec: + host: cafe.example.com + . . . + ``` + +If a user creates both resources in the cluster, a host collision will occur. NGINX Ingress Controller will pick the winner using the [winner selection algorithm](#winner-selection-algorithm). + +If `cafe-virtual-server` was created first, it will win the host `cafe.example.com` and NGINX Ingress Controller will reject `cafe-ingress`. This will be reflected in the events and in the resource's status field: + +```shell +kubectl describe vs cafe-virtual-server +``` +```text +... +Status: + ... + Message: Configuration for default/cafe-virtual-server was added or updated + Reason: AddedOrUpdated + State: Valid +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 9s nginx-ingress-controller Configuration for default/cafe-virtual-server was added or updated +``` + +```shell +kubectl describe ingress cafe-ingress +``` +```text +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 66s nginx-ingress-controller All hosts are taken by other resources +``` + +Similarly, if `cafe-ingress` was created first, it will win `cafe.example.com` and NGINX Ingress Controller will reject `cafe-virtual-server`. + +{{< note >}} You can configure multiple hosts for Ingress resources, and its possible that an Ingress resource can be the winner for some of its hosts and a loser for the others. + +For example, if `cafe-ingress` had an additional rule host rule for `pub.example.com`, NGINX Ingress Controller would not reject the Ingress. Instead, it would allow `cafe-ingress` to handle `pub.example.com`. {{< /note >}} + +--- + +### Merging configuration for the same host + +It is possible to merge configuration for multiple Ingress resources for the same host. One common use case for this approach is distributing resources across multiple namespaces. + +The [Cross-namespace configuration]({{< ref "/nic/configuration/ingress-resources/cross-namespace-configuration.md">}}) topic has more information. + +It is *not* possible to merge the configurations for multiple VirtualServer resources for the same host. However, you can split the VirtualServers into multiple VirtualServerRoute resources, which a single VirtualServer can then reference. See the [corresponding example](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/cross-namespace-configuration) on GitHub. + +It is *not* possible to merge configuration for multiple TransportServer resources. + +--- + +## Listener/Host collisions + +Listener/Host collisions occur when multiple TransportServer resources (configured for TCP/UDP load balancing) specify the same combination of `spec.listener.name` and `spec.host`. + +The combination of `spec.listener.name` and `spec.host` must be unique among all TransportServer resources. If two TransportServer resources specify the same spec.listener.name and spec.host, one of them will be rejected to prevent conflicts. In the case where spec.host is not specified, it is considered an empty string. + +NGINX Ingress Controller will choose the winner, which will own that listener and host combination. + +--- + +### Choosing the winner + +Consider the following two resources: + +- `tcp-1` TransportServer: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: tcp-1 + spec: + host: dns.example.com + listener: + name: dns-tcp + protocol: TCP + . . . + ``` + +- `tcp-2` TransportServer: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: tcp-2 + spec: + host: dns.example.com + listener: + name: dns-tcp + protocol: TCP + . . . + ``` + +If a user creates both resources in the cluster, a listener collision will occur. As a result, NGINX Ingress Controller will pick the winner using the [winner selection algorithm](#winner-selection-algorithm). + +In our example, if `tcp-1` was created first, it will win the listener `dns-tcp` and NGINX Ingress Controller will reject `tcp-2`. This will be reflected in the events and in the resource's status field: + +```shell +kubectl describe ts tcp-2 +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 10s nginx-ingress-controller Listener dns-tcp is taken by another resource +``` + +Similarly, if `tcp-2` was created first, it will win `dns-tcp` and NGINX Ingress Controller will reject `tcp-1`. diff --git a/content/nic/configuration/ingress-resources/_index.md b/content/nic/configuration/ingress-resources/_index.md new file mode 100644 index 000000000..8a2b5b282 --- /dev/null +++ b/content/nic/configuration/ingress-resources/_index.md @@ -0,0 +1,8 @@ +--- +title: Ingress resources +description: +weight: 200 +menu: + docs: + parent: NGINX Ingress Controller +--- diff --git a/content/nic/configuration/ingress-resources/advanced-configuration-with-annotations.md b/content/nic/configuration/ingress-resources/advanced-configuration-with-annotations.md new file mode 100644 index 000000000..ee3dbc860 --- /dev/null +++ b/content/nic/configuration/ingress-resources/advanced-configuration-with-annotations.md @@ -0,0 +1,225 @@ +--- +docs: DOCS-591 +doctypes: +- '' +title: Advanced configuration with Annotations +toc: true +weight: 200 +--- + +This topic explains how to enable advanced features in F5 NGINX Ingress Controller with Annotations. + +The Ingress resource can use basic NGINX features such as host or path-based routing and TLS termination. Advanced features like rewriting the request URI or inserting additional response headers can be enabled with Annotations. + +Outside of advanced features, Annotations are necessary for customizing NGINX behavior such as setting the value of connection timeouts. + +Customization is also available through the [ConfigMap]({{< ref "/nic/configuration/global-configuration/configmap-resource.md" >}}) resources: Annotations take priority. + +## Using Annotations + +This example uses Annotations to customize the configuration for an Ingress resource: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: cafe-ingress-with-annotations + annotations: + nginx.org/proxy-connect-timeout: "30s" + nginx.org/proxy-read-timeout: "20s" + nginx.org/client-max-body-size: "4m" + nginx.org/server-snippets: | + location / { + return 302 /coffee; + } +spec: + rules: + - host: cafe.example.com + http: + paths: + - path: /tea + pathType: Prefix + backend: + service: + name: tea-svc + port: + number: 80 + - path: /coffee + pathType: Prefix + backend: + service: + name: coffee-svc + port: + number: 80 +``` + +## Validation + +NGINX Ingress Controller validates the annotations of Ingress resources. If an Ingress is invalid, NGINX Ingress Controller will reject it: the Ingress will continue to exist in the cluster, but NGINX Ingress Controller will ignore it. + +You can check if NGINX Ingress Controller successfully applied the configuration for an Ingress resource. For the example `cafe-ingress-with-annotations` Ingress, you can run: + +```shell +kubectl describe ing cafe-ingress-with-annotations +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 3s nginx-ingress-controller Configuration for default/cafe-ingress-with-annotations was added or updated +``` + +The events section includes a Normal event with the AddedOrUpdated reason that informs us that the configuration was successfully applied. + +If you create an invalid Ingress, NGINX Ingress Controller will reject it and emit a Rejected event. For example, if you create an Ingress `cafe-ingress-with-annotations`, with an annotation `nginx.org/redirect-to-https` set to `yes please` instead of `true`, you will get: + +```shell +kubectl describe ing cafe-ingress-with-annotations +``` +```text +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 13s nginx-ingress-controller annotations.nginx.org/redirect-to-https: Invalid value: "yes please": must be a boolean +``` + +Note how the events section includes a Warning event with the Rejected reason. + +{{< note >}} If you make an existing Ingress invalid, NGINX Ingress Controller will reject it and remove the corresponding configuration from NGINX. {{< /note >}} + +The `nginx.com/jwt-token` Ingress annotation has limited validation. + +## Summary of Annotations + +The table below summarizes the available annotations. + +{{< note >}} Annotations that start with `nginx.com` are only supported with NGINX Plus. {{< /note >}} + +### General customization + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/proxy-connect-timeout* | *proxy-connect-timeout* | Sets the value of the [proxy_connect_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) and [grpc_connect_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout) directive. | *60s* | | +| *nginx.org/proxy-read-timeout* | *proxy-read-timeout* | Sets the value of the [proxy_read_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) and [grpc_read_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout) directive. | *60s* | | +| *nginx.org/proxy-send-timeout* | *proxy-send-timeout* | Sets the value of the [proxy_send_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout) and [grpc_send_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_send_timeout) directive. | *60s* | | +| *nginx.org/client-max-body-size* | *client-max-body-size* | Sets the value of the [client_max_body_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) directive. | *1m* | | +| *nginx.org/proxy-buffering* | *proxy-buffering* | Enables or disables [buffering of responses](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) from the proxied server. | *True* | | +| *nginx.org/proxy-buffers* | *proxy-buffers* | Sets the value of the [proxy_buffers](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directive. | Depends on the platform. | | +| *nginx.org/proxy-buffer-size* | *proxy-buffer-size* | Sets the value of the [proxy_buffer_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) and [grpc_buffer_size](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_buffer_size) directives. | Depends on the platform. | | +| *nginx.org/proxy-max-temp-file-size* | *proxy-max-temp-file-size* | Sets the value of the [proxy_max_temp_file_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size) directive. | *1024m* | | +| *nginx.org/server-tokens* | *server-tokens* | Enables or disables the [server_tokens](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens) directive. Additionally, with the NGINX Plus, you can specify a custom string value, including the empty string value, which disables the emission of the “Server” field. | *True* | | +| *nginx.org/path-regex* | N/A | Enables regular expression modifiers for Ingress path parameter. This translates to the NGINX [location](https://nginx.org/en/docs/http/ngx_http_core_module.html#location) directive. You can specify one of these values: "case_sensitive", "case_insensitive", or "exact". The annotation is applied to the entire Ingress resource and its paths. While using Master and Minion Ingresses i.e. Mergeable Ingresses, this annotation can be specified on Minion types. The `path-regex` annotation specified on Master is ignored, and has no effect on paths defined on Minions. | N/A | [path-regex](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/path-regex) | +{{}} + +### Request URI/Header Manipulation + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/proxy-hide-headers* | *proxy-hide-headers* | Sets the value of one or more [proxy_hide_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directives. Example: ``"nginx.org/proxy-hide-headers": "header-a,header-b"* | N/A | | +| *nginx.org/proxy-pass-headers* | *proxy-pass-headers* | Sets the value of one or more [proxy_pass_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directives. Example: ``"nginx.org/proxy-pass-headers": "header-a,header-b"* | N/A | | +| *nginx.org/rewrites* | N/A | Configures URI rewriting using [proxy_pass](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass) directive. | N/A | [rewrites](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/rewrites) | +|*nginx.org/proxy-set-headers* | N/A | Enables customization of proxy headers and values using the [proxy_set_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) directive. Example: ``"nginx.org/proxy-set-headers": "header-a: valueA,header-b: valueB,header-c: valueC"`` | N/A | [Proxy Set Headers](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/proxy-set-headers). | +{{}} + +### Auth and SSL/TLS + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/redirect-to-https* | *redirect-to-https* | Sets the 301 redirect rule based on the value of the ``http_x_forwarded_proto* header on the server block to force incoming traffic to be over HTTPS. Useful when terminating SSL in a load balancer in front of NGINX Ingress Controller — see [115](https://github.com/nginx/kubernetes-ingress/issues/115) | *False* | | +| *ingress.kubernetes.io/ssl-redirect* | *ssl-redirect* | Sets an unconditional 301 redirect rule for all incoming HTTP traffic to force incoming traffic over HTTPS. | *True* | | +| *nginx.org/hsts* | *hsts* | Enables [HTTP Strict Transport Security (HSTS)](https://www.nginx.com/blog/http-strict-transport-security-hsts-and-nginx/)\ : the HSTS header is added to the responses from backends. The ``preload* directive is included in the header. | *False* | | +| *nginx.org/hsts-max-age* | *hsts-max-age* | Sets the value of the ``max-age* directive of the HSTS header. | *2592000* (1 month) | | +| *nginx.org/hsts-include-subdomains* | *hsts-include-subdomains* | Adds the ``includeSubDomains* directive to the HSTS header. | *False* | | +| *nginx.org/hsts-behind-proxy* | *hsts-behind-proxy* | Enables HSTS based on the value of the ``http_x_forwarded_proto* request header. Should only be used when TLS termination is configured in a load balancer (proxy) in front of NGINX Ingress Controller. Note: to control redirection from HTTP to HTTPS configure the ``nginx.org/redirect-to-https* annotation. | *False* | | +| *nginx.org/basic-auth-secret* | N/A | Specifies a Secret resource with a user list for HTTP Basic authentication. | N/A | | +| *nginx.org/basic-auth-realm* | N/A | Specifies a realm. | N/A | | +| *nginx.com/jwt-key* | N/A | Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/jwt). | +| *nginx.com/jwt-realm* | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/jwt). | +| *nginx.com/jwt-token* | N/A | Specifies a variable that contains a JSON Web Token. | By default, a JWT is expected in the ``Authorization* header as a Bearer Token. | [Support for JSON Web Tokens (JWTs)](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/jwt). | +| *nginx.com/jwt-login-url* | N/A | Specifies a URL to which a client is redirected in case of an invalid or missing JWT. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/jwt). | +{{}} + +### Listeners + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/listen-ports* | N/A | Configures HTTP ports that NGINX will listen on. | *[80]* | | +| *nginx.org/listen-ports-ssl* | N/A | Configures HTTPS ports that NGINX will listen on. | *[443]* | | +{{}} + +### Backend services (Upstreams) + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/lb-method* | *lb-method* | Sets the [load balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify ``"round_robin"``. | *"random two least_conn"* | | +| *nginx.org/ssl-services* | N/A | Enables HTTPS or gRPC over SSL when connecting to the endpoints of services. | N/A | [ssl-services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/ssl-services) | +| *nginx.org/grpc-services* | N/A | Enables gRPC for services. Note: requires HTTP/2 (see ``http2* ConfigMap key); only works for Ingresses with TLS termination enabled. | N/A | [grpc-services](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/grpc-services) | +| *nginx.org/websocket-services* | N/A | Enables WebSocket for services. | N/A | [websocket](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/websocket) | +| *nginx.org/max-fails* | *max-fails* | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the ``server* directive. | *1* | | +| *nginx.org/max-conns* | N\A | Sets the value of the [max_conns](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_conns) parameter of the ``server* directive. | *0* | | +| *nginx.org/upstream-zone-size* | *upstream-zone-size* | Sets the size of the shared memory [zone](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#zone) for upstreams. For NGINX, the special value 0 disables the shared memory zones. For NGINX Plus, shared memory zones are required and cannot be disabled. The special value 0 will be ignored. | *256K* | | +| *nginx.org/fail-timeout* | *fail-timeout* | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the ``server* directive. | *10s* | | +| *nginx.com/sticky-cookie-services* | N/A | Configures session persistence. | N/A | [session-persistence](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/session-persistence) | +| *nginx.org/keepalive* | *keepalive* | Sets the value of the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. Note that ``proxy_set_header Connection "";* is added to the generated configuration when the value > 0. | *0* | | +| *nginx.com/health-checks* | N/A | Enables active health checks. | *False* | [health-checks](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/health-checks) | +| *nginx.com/health-checks-mandatory* | N/A | Configures active health checks as mandatory. | *False* | [health-checks](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/health-checks) | +| *nginx.com/health-checks-mandatory-queue* | N/A | When active health checks are mandatory, creates a queue where incoming requests are temporarily stored while NGINX Plus is checking the health of the endpoints after a configuration reload. | *0* | [health-checks](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/health-checks) | +| *nginx.com/slow-start* | N/A | Sets the upstream server [slow-start period](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#server-slow-start). By default, slow-start is activated after a server becomes [available](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#passive-health-checks) or [healthy](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#active-health-checks). To enable slow-start for newly-added servers, configure [mandatory active health checks](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/health-checks). | *"0s"* | | +| *nginx.org/use-cluster-ip* | N/A | Enables using the Cluster IP and port of the service instead of the default behavior of using the IP and port of the pods. When this field is enabled, the fields that configure NGINX behavior related to multiple upstream servers (like ``lb-method* and ``next-upstream``) will have no effect, as NGINX Ingress Controller will configure NGINX with only one upstream server that will match the service Cluster IP. | *False* | | +{{}} + +### Rate limiting + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/limit-req-rate* | N/A | Enables request-rate-limiting for this ingress by creating a [limit_req_zone](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone) and matching [limit_req](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) for each location. All servers/locations of one ingress share the same zone. Must have unit r/s or r/m. | N/A | 200r/s | +| *nginx.org/limit-req-key* | N/A | The key to which the rate limit is applied. Can contain text, variables, or a combination of them. Variables must be surrounded by ${}. | ${binary_remote_addr} | ${binary_remote_addr} | +| *nginx.org/limit-req-zone-size* | N/A | Configures the size of the created [limit_req_zone](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone). | 10m | 20m | +| *nginx.org/limit-req-delay* | N/A | Configures the delay-parameter of the [limit_req](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) directive. | 0 | 100 | +| *nginx.org/limit-req-no-delay* | N/A | Configures the nodelay-parameter of the [limit_req](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) directive. | false | true | +| *nginx.org/limit-req-burst* | N/A | Configures the burst-parameter of the [limit_req](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) directive. | N/A | 100 | +| *nginx.org/limit-req-dry-run* | N/A | Enables the dry run mode. In this mode, the rate limit is not actually applied, but the number of excessive requests is accounted as usual in the shared memory zone. | false | true | +| *nginx.org/limit-req-log-level* | N/A | Sets the desired logging level for cases when the server refuses to process requests due to rate exceeding, or delays request processing. Allowed values are info, notice, warn or error. | error | info | +| *nginx.org/limit-req-reject-code* | N/A | Sets the status code to return in response to rejected requests. Must fall into the range 400..599. | 429 | 503 | +| *nginx.org/limit-req-scale* | N/A | Enables a constant rate-limit by dividing the configured rate by the number of nginx-ingress pods currently serving traffic. This adjustment ensures that the rate-limit remains consistent, even as the number of nginx-pods fluctuates due to autoscaling. Note: This will not work properly if requests from a client are not evenly distributed accross all ingress pods (sticky sessions, long lived TCP-Connections with many requests etc.). In such cases using [zone-sync]({{< ref "/nic/configuration/global-configuration/configmap-resource.md#zone-sync" >}}) instead would give better results. Enabling `zone-sync` will suppress this setting. | false | true | +{{}} + +### Snippets and custom templates + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *nginx.org/location-snippets* | *location-snippets* | Sets a custom snippet in location context. | N/A | | +| *nginx.org/server-snippets* | *server-snippets* | Sets a custom snippet in server context. | N/A | | +{{}} + +### App Protect WAF {#app-protect} + +{{< note >}} The App Protect annotations only work if the App Protect WAF module is [installed]({{< ref "/nic/installation/integrations/app-protect-waf/installation.md" >}}). {{< /note >}} + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *appprotect.f5.com/app-protect-policy* | N/A | The name of the App Protect Policy for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace of the Ingress Resource is used. If not specified but ``appprotect.f5.com/app-protect-enable* is true, a default policy id applied. If the referenced policy resource does not exist, or policy is invalid, this annotation will be ignored, and the default policy will be applied. | N/A | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +| *appprotect.f5.com/app-protect-enable* | N/A | Enable App Protect for the Ingress Resource. | *False* | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +| *appprotect.f5.com/app-protect-security-log-enable* | N/A | Enable the [security log](/nginx-app-protect/troubleshooting/#app-protect-logging-overview) for App Protect. | *False* | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +| *appprotect.f5.com/app-protect-security-log* | N/A | The App Protect log configuration for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace as the Ingress Resource is used. If not specified the default is used which is: filter: ``illegal``, format: ``default``. Multiple configurations can be specified in a comma separated list. Both log configurations and destinations list (see below) must be of equal length. Configs and destinations are paired by the list indices. | N/A | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +| *appprotect.f5.com/app-protect-security-log-destination* | N/A | The destination of the security log. For more information check the [DESTINATION argument](/nginx-app-protect/troubleshooting/#app-protect-logging-overview). Multiple destinations can be specified in a comma-separated list. Both log configurations and destinations list (see above) must be of equal length. Configs and destinations are paired by the list indices. | *syslog:server=localhost:514* | [app-protect-waf](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-waf) | +{{}} + +### App Protect DoS + +{{< note >}} The App Protect DoS annotations only work if the App Protect DoS module is [installed]({{< ref "/nic/installation/integrations/app-protect-dos/installation.md" >}}). {{< /note >}} + +{{}} +|Annotation | ConfigMap Key | Description | Default | Example | +| ---| ---| ---| ---| --- | +| *appprotectdos.f5.com/app-protect-dos-resource* | N/A | Enable App Protect DoS for the Ingress Resource by specifying a [DosProtectedResource]({{< ref "/nic/installation/integrations/app-protect-dos/dos-protected.md" >}}). | N/A | [app-protect-dos](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/app-protect-dos) | +{{}} diff --git a/content/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md b/content/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md new file mode 100644 index 000000000..0060a7e3b --- /dev/null +++ b/content/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md @@ -0,0 +1,128 @@ +--- +docs: DOCS-592 +doctypes: +- '' +title: Advanced configuration with Snippets +toc: true +weight: 400 +--- + +Snippets allow you to insert raw NGINX config into different contexts of the NGINX configurations that F5 NGINX Ingress Controller generates. + +Snippets are intended for advanced NGINX users who need more control over the generated NGINX configuration, and can be used in cases where Annotations and ConfigMap entries would not apply. + + + +## Disadvantages of snippets + +Snippets are configured [using Annotations]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-annotations.md#snippets-and-custom-templates" >}}), but are disabled by default due to their complexity. They are also available through the [ConfigMap]({{< ref "/nic/configuration/global-configuration/configmap-resource.md#snippets-and-custom-templates" >}}) resource. + +To use snippets, set the [`enable-snippets`]({{< ref "/nic/configuration/global-configuration/command-line-arguments.md#cmdoption-enable-snippets" >}}) command-line argument. + +Snippets have the following disadvantages: + +- *Complexity*. Snippets require you to: + - Understand NGINX configuration primitives and implement a correct NGINX configuration. + - Understand how NGINX Ingress Controller generates NGINX configuration so that a snippet doesn't interfere with the other features in the configuration. +- *Decreased robustness*. An incorrect snippet can invalidate NGINX configuration, causing reload failures. Until the snippet is fixed, it will prevent any new configuration updates, including updates for the other Ingress resources. +- *Security implications*. Snippets give access to NGINX configuration primitives, which are not validated by NGINX Ingress Controller. For example, a snippet can configure NGINX to serve the TLS certificates and keys used for TLS termination for Ingress resources. + +{{< note >}} If the NGINX configuration includes an invalid snippet, NGINX will continue to operate with the last valid configuration. {{< /note >}} + +## Using snippets + +The example below shows how to use snippets to customize the NGINX configuration template using annotations. + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: cafe-ingress-with-snippets + annotations: + nginx.org/server-snippets: | + location / { + return 302 /coffee; + } + nginx.org/location-snippets: | + add_header my-test-header test-value; +spec: + rules: + - host: cafe.example.com + http: + paths: + - path: /tea + pathType: Prefix + backend: + service: + name: tea-svc + port: + number: 80 + - path: /coffee + pathType: Prefix + backend: + service: + name: coffee-svc + port: + number: 80 +``` + +These snippets generate the following NGINX configuration: + +{{< note >}} The example is shortened for conciseness. {{< /note >}} + +```nginx +server { + listen 80; + + + location / { + return 302 /coffee; + } + + + location /coffee { + proxy_http_version 1.1; + + + add_header my-test-header test-value; + ... + proxy_pass http://default-cafe-ingress-with-snippets-cafe.example.com-coffee-svc-80; + } + + location /tea { + proxy_http_version 1.1; + + add_header my-test-header test-value; + ... + proxy_pass http://default-cafe-ingress-with-snippets-cafe.example.com-tea-svc-80; + } +} +``` + +## Troubleshooting + +If a snippet includes an invalid NGINX configuration, NGINX Ingress Controller will fail to reload NGINX. The error will be reported in NGINX Ingress Controller logs and an event with the error will be associated with the Ingress resource: + +An example of an error from the logs: + +```text +[emerg] 31#31: unknown directive "badd_header" in /etc/nginx/conf.d/default-cafe-ingress-with-snippets.conf:54 +Event(v1.ObjectReference{Kind:"Ingress", Namespace:"default", Name:"cafe-ingress-with-snippets", UID:"f9656dc9-63a6-41dd-a499-525b0e0309bb", APIVersion:"extensions/v1beta1", ResourceVersion:"2322030", FieldPath:""}): type: 'Warning' reason: 'AddedOrUpdatedWithError' Configuration for default/cafe-ingress-with-snippets was added or updated, but not applied: Error reloading NGINX for default/cafe-ingress-with-snippets: nginx reload failed: Command /usr/sbin/nginx -s reload stdout: "" +stderr: "nginx: [emerg] unknown directive \"badd_header\" in /etc/nginx/conf.d/default-cafe-ingress-with-snippets.conf:54\n" +finished with error: exit status 1 +``` + +An example of an event with an error (you can view events associated with the Ingress by running `kubectl describe -n nginx-ingress ingress nginx-ingress`): + +```text +Events: +Type Reason Age From Message +---- ------ ---- ---- ------- +Normal AddedOrUpdated 52m (x3 over 61m) nginx-ingress-controller Configuration for default/cafe-ingress-with-snippets was added or updated +finished with error: exit status 1 +Warning AddedOrUpdatedWithError 54s (x2 over 89s) nginx-ingress-controller Configuration for default/cafe-ingress-with-snippets was added or updated, but not applied: Error reloading NGINX for default/cafe-ingress-with-snippets: nginx reload failed: Command /usr/sbin/nginx -s reload stdout: "" +stderr: "nginx: [emerg] unknown directive \"badd_header\" in /etc/nginx/conf.d/default-cafe-ingress-with-snippets.conf:54\n" +finished with error: exit status 1 +``` + +Additionally, to help troubleshoot snippets, a number of Prometheus metrics show the stats about failed reloads – `controller_nginx_last_reload_status` and `controller_nginx_reload_errors_total`. diff --git a/content/nic/configuration/ingress-resources/basic-configuration.md b/content/nic/configuration/ingress-resources/basic-configuration.md new file mode 100644 index 000000000..8d1948693 --- /dev/null +++ b/content/nic/configuration/ingress-resources/basic-configuration.md @@ -0,0 +1,107 @@ +--- +title: Basic configuration +weight: 100 +toc: true +type: reference +product: NIC +docs: DOCS-593 +--- + +This document shows a basic Ingress resource definition for F5 NGINX Ingress Controller. It load balances requests for two services as part of a single application. + +{{< ghcode `https://raw.githubusercontent.com/nginx/kubernetes-ingress/refs/heads/main/examples/ingress-resources/complete-example/cafe-ingress.yaml`>}} + +Here is a breakdown of what this Ingress resource definition means: + +- The `metadata.name` field defines the name of the resource `cafe‑ingress`. +- The `spec.tls` field sets up SSL/TLS termination: + - The `hosts` field applies the certificate and key to the `cafe.example.com` host. + - The `secretName` references a secret resource by its name, `cafe‑secret`. The secret must belong to the same namespace as the Ingress, of the type ``kubernetes.io/tls`` and contain keys named ``tls.crt`` and ``tls.key`` that hold the certificate and private key as described [here](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls>). If the secret doesn't exist or is invalid, NGINX will break any attempt to establish a TLS connection to the hosts to which the secret is applied. +- The `spec.rules` field defines a host with the domain name `cafe.example.com`. +- The `paths` field defines two path‑based rules: + - The rule with the path `/tea` instructs NGINX to distribute the requests with the `/tea` URI among the pods of the *tea* service, which is deployed with the name `tea‑svc` in the cluster. + - The rule with the path `/coffee` instructs NGINX to distribute the requests with the `/coffee` URI among the pods of the *coffee* service, which is deployed with the name `coffee‑svc` in the cluster. + - Both rules instruct NGINX to distribute the requests to `port 80` of the corresponding service (the `servicePort` field). + +To learn more about the Ingress resource, view [the official Kubernetes documentation for Ingress resources](https://kubernetes.io/docs/concepts/services-networking/ingress/). + +{{< note >}} For complete instructions on deploying Ingress and Secret resources in the cluster, see the [complete example](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/complete-example) in the GitHub repository. {{< /note >}} + +--- + +## New features available in Kubernetes 1.18 + +Starting from Kubernetes 1.18, you can use the following new features: + +- The host field supports wildcard domain names, such as `*.example.com`. +- The path supports different matching rules with the new field `pathType`, which takes the following values: `Prefix` for prefix-based matching, `Exact` for exact matching and `ImplementationSpecific`, which is the default type and is the same as `Prefix`. For example: + + ```yaml {hl_lines=[2, 7, 14]} + - path: /tea + pathType: Prefix + backend: + serviceName: tea-svc + servicePort: 80 + - path: /tea/green + pathType: Exact + backend: + service: + name: tea-svc + port: + number: 80 + - path: /coffee + pathType: ImplementationSpecific + backend: + service: + name: coffee-svc + port: + number: 80 + ``` + +- The `ingressClassName` field is now supported: + + ```yaml {hl_lines=[6]} + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: cafe-ingress + spec: + ingressClassName: nginx + tls: + - hosts: + - cafe.example.com + secretName: cafe-secret + rules: + - host: cafe.example.com + . . . + ``` + + When using this field you need to create the `IngressClass` resource with the corresponding `name`. View the [Create common resources]({{< ref "/nic/installation/installing-nic/installation-with-manifests.md#create-common-resources" >}}) section of the Installation with Manifests topic for more information. + +--- + +## Restrictions + +NGINX Ingress Controller imposes the following restrictions on Ingress resources: + +- When defining an Ingress resource, the `host` field is required. +- The `host` value needs to be unique among all Ingress and VirtualServer resources unless the Ingress resource is a [mergeable minion]({{< ref "/nic/configuration/ingress-resources/cross-namespace-configuration.md" >}}). View the [Host and Listener collisions]({{< ref "/nic/configuration/host-and-listener-collisions.md" >}}) topic for more information. +- The `path` field in `spec.rules[].http.paths[]` is required for `Exact` and `Prefix` `pathTypes`. +- The ImplementationSpecific `pathType` is treated as equivalent to `Prefix` `pathType`, with the exception that when this `pathType` is configured, the `path` field in `spec.rules[].http.paths[]` is not mandatory. `path` defaults to `/` if not set but the `pathType` is set to ImplementationSpecific. + +--- + +## Advanced configuration + +NGINX Ingress Controller generates NGINX configuration by executing a template file that contains the configuration options. + +These options are set with the Ingress resource and NGINX Ingress Controller's ConfigMap. + +The Ingress resource only allows you to use basic NGINX features: host and path-based routing and TLS termination. + +For advanced configuration, you have two options: + +- [Annotations]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-annotations.md" >}}) can be used to rewrite request URIs or inserting additional response headers. +- [Snippets]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-snippets" >}}) can be used to insert raw NGINX configuration, changing generated files. + +Additionally, it is possible to customize the template, described in the [Custom templates]({{< ref "/nic/configuration/global-configuration/custom-templates.md" >}}) topic. diff --git a/content/nic/configuration/ingress-resources/cross-namespace-configuration.md b/content/nic/configuration/ingress-resources/cross-namespace-configuration.md new file mode 100644 index 000000000..3e92b6403 --- /dev/null +++ b/content/nic/configuration/ingress-resources/cross-namespace-configuration.md @@ -0,0 +1,14 @@ +--- +docs: DOCS-594 +doctypes: +- '' +title: Cross-namespace configuration +toc: true +weight: 500 +--- + +This topic explains how to spread Ingress configuration across different namespaces in F5 NGINX Ingress Controller. + +You can spread the Ingress configuration for a common host across multiple Ingress resources using Mergeable Ingress resources. Such resources can belong to the *same* or *different* namespaces. This enables easier management when using a large number of paths. See the [Mergeable Ingress Resources](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/mergeable-ingress-types) example in our GitHub repo. + +As an alternative to Mergeable Ingress resources, you can use [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/) for cross-namespace configuration. See the [Cross-Namespace Configuration](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/cross-namespace-configuration) example in our GitHub repo. diff --git a/content/nic/configuration/ingress-resources/custom-annotations.md b/content/nic/configuration/ingress-resources/custom-annotations.md new file mode 100644 index 000000000..8bbd136b0 --- /dev/null +++ b/content/nic/configuration/ingress-resources/custom-annotations.md @@ -0,0 +1,146 @@ +--- +docs: DOCS-595 +doctypes: +- '' +title: Custom annotations +toc: true +weight: 300 +--- + +This topic explains how you can use custom annotations with F5 NGINX Ingress Controller. + +Custom annotations enable you to quickly extend the Ingress resource to support many advanced features of NGINX, such as rate limiting, caching, etc. + +## Overview + +NGINX Ingress Controller supports a number of annotations for the Ingress resource that fine tune NGINX configuration (for example, connection timeouts) or enable additional features (for example, JWT validation). The complete list of annotations is available [here](/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations). + +The annotations are provided only for the most common features and use cases, meaning that not every NGINX feature or a customization option is available through the annotations. Additionally, even if an annotation is available, it might not give you the satisfactory level of control of a particular NGINX feature. + +Custom annotations allow you to add an annotation for an NGINX feature that is not available as a regular annotation. In contrast with regular annotations, to add a custom annotation, you don't need to modify the Ingress Controller source code -- just modify the template. Additionally, with a custom annotation, you get full control of how the feature is implemented in NGINX configuration. + +## Usage + +The Ingress Controller generates NGINX configuration for Ingress resources by executing a configuration template. See [NGINX template](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/internal/configs/version1/nginx.ingress.tmpl) or [NGINX Plus template](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/internal/configs/version1/nginx-plus.ingress.tmpl). + +To support custom annotations, the template has access to the information about the Ingress resource - its *name*, *namespace* and *annotations*. It is possible to check if a particular annotation present in the Ingress resource and conditionally insert NGINX configuration directives at multiple NGINX contexts - `http`, `server`, `location` or `upstream`. Additionally, you can get the value that is set to the annotation. + +Consider the following excerpt from the template, which was extended to support two custom annotations: + +```jinja2 +# This is the configuration for {{$.Ingress.Name}}/{{$.Ingress.Namespace}} + +{{if index $.Ingress.Annotations "custom.nginx.org/feature-a"}} +# Insert config for feature A if the annotation is set +{{end}} + +{{with $value := index $.Ingress.Annotations "custom.nginx.org/feature-b"}} +# Insert config for feature B if the annotation is set +# Print the value assigned to the annotation: {{$value}} +{{end}} +``` + +Consider the following Ingress resource and note how we set two annotations: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: example-ingress + namespace: production + annotations: + custom.nginx.org/feature-a: "on" + custom.nginx.org/feature-b: "512" +spec: + rules: + - host: example.com + . . . +``` + +Assuming that the Ingress Controller is using that customized template, it will generate a config for the Ingress resource that will include the following part, generated by our template excerpt: + +```yaml +# This is the configuration for cafe-ingress/default + +# Insert config for feature A if the annotation is set + + + +# Insert config for feature B if the annotation is set +# Print the value assigned to the annotation: 512 +``` + +**Notes**: + +- You can customize the template to insert you custom annotations via [custom templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). +- The Ingress Controller uses go templates to generate NGINX config. You can read more information about go templates [here](https://golang.org/pkg/text/template/). + +See the examples in the next section that use custom annotations to configure NGINX features. + +### Custom Annotations with Mergeable Ingress Resources + +A Mergeable Ingress resource consists of multiple Ingress resources - one master and one or several minions. Read more about Mergeable Ingress resources [here](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration). + +If you'd like to use custom annotations with Mergeable Ingress resources, please keep the following in mind: + +- Custom annotations can be used in the Master and in Minions. For Minions, you can access them in the template only when processing locations. + + If you access `$.Ingress` anywhere in the Ingress template, you will get the master Ingress resource. To access a Minion Ingress resource, use `$location.MinionIngress`. However, it is only available when processing locations: + + ```jinja2 + {{range $location := $server.Locations}} + location {{$location.Path}} { + {{with $location.MinionIngress}} + # location for minion {{$location.MinionIngress.Namespace}}/{{$location.MinionIngress.Name}} + {{end}} + } {{end}} + ``` + + **Note**: `$location.MinionIngress` is a pointer. When a regular Ingress resource is processed in the template, the value of the pointer is `nil`. Thus, it is important that you check that `$location.MinionIngress` is not `nil` as in the example above using the `with` action. + +- Minions do not inherent custom annotations of the master. + +### Helper Functions + +Helper functions can be used in the Ingress template to parse the values of custom annotations. + +{{% table %}} +| Function | Input Arguments | Return Arguments | Description | +| ---| ---| ---| --- | +| ``split`` | ``s, sep string`` | ``[]string`` | Splits the string ``s`` into a slice of strings separated by the ``sep``. | +| ``trim`` | ``s string`` | ``string`` | Trims the trailing and leading whitespace from the string ``s``. | +| ``contains`` | ``s, substr string`` | ``bool`` | Tests whether the string ``substr`` is a substring of the string ``s``. | +| ``hasPrefix`` | ``s, prefix string`` | ``bool`` | Tests whether the string ``prefix`` is a prefix of the string ``s``. | +| ``hasSuffix`` | ``s, suffix string`` | ``bool`` | Tests whether the string ``suffix`` is a suffix of the string ``s``. | +| ``toLower`` | ``s string`` | ``bool`` | Converts all letters in the string ``s`` to their lower case. | +| ``toUpper`` | ``s string`` | ``bool`` | Converts all letters in the string ``s`` to their upper case. | +| ``replaceAll`` | ``s, old, new string`` | ``string`` | Replaces all occurrences of ``old`` with ``new`` in the string ``s``. | +{{% /table %}} + +Consider the following custom annotation `custom.nginx.org/allowed-ips`, which expects a comma-separated list of IP addresses: + +```yaml +annotations: + custom.nginx.org/allowed-ips: "192.168.1.3, 10.0.0.13" +``` + + The helper functions can parse the value of the `custom.nginx.org/allowed-ips` annotation, so that in the template you can use each IP address separately. Consider the following template excerpt: + +```jinja2 +{{range $ip := split (index $.Ingress.Annotations "custom.nginx.org/allowed-ips") ","}} + allow {{trim $ip}}; +{{end}} +deny all; +``` + +The template excerpt will generate the following configuration: + +``` +allow 192.168.1.3; +allow 10.0.0.13; +deny all; +``` + +## Example + +See the [custom annotations example](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/examples/ingress-resources/custom-annotations). diff --git a/content/nic/configuration/policy-resource.md b/content/nic/configuration/policy-resource.md new file mode 100644 index 000000000..4188188ef --- /dev/null +++ b/content/nic/configuration/policy-resource.md @@ -0,0 +1,947 @@ +--- +title: Policy resources +weight: 500 +toc: true +type: how-to +product: NIC +docs: DOCS-596 +--- + +The Policy resource allows you to configure features like access control and rate-limiting, which you can add to your [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/). + +The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + +This document is the reference documentation for the Policy resource. An example of a Policy for access control is available in our [GitHub repository](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/examples/custom-resources/access-control). + +## Prerequisites + +Policies work together with [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/), which you need to create separately. + +## Policy Specification + +Below is an example of a policy that allows access for clients from the subnet `10.0.0.0/8` and denies access for any other clients: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: Policy +metadata: + name: allow-localhost +spec: + accessControl: + allow: + - 10.0.0.0/8 +``` + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``accessControl`` | The access control policy based on the client IP address. | [accessControl](#accesscontrol) | No | +|``ingressClassName`` | Specifies which instance of NGINX Ingress Controller must handle the Policy resource. | ``string`` | No | +|``rateLimit`` | The rate limit policy controls the rate of processing requests per a defined key. | [rateLimit](#ratelimit) | No | +|``apiKey`` | The API Key policy configures NGINX to authorize requests which provide a valid API Key in a specified header or query param. | [apiKey](#apikey) | No | +|``basicAuth`` | The basic auth policy configures NGINX to authenticate client requests using HTTP Basic authentication credentials. | [basicAuth](#basicauth) | No | +|``jwt`` | The JWT policy configures NGINX Plus to authenticate client requests using JSON Web Tokens. | [jwt](#jwt) | No | +|``ingressMTLS`` | The IngressMTLS policy configures client certificate verification. | [ingressMTLS](#ingressmtls) | No | +|``egressMTLS`` | The EgressMTLS policy configures upstreams authentication and certificate verification. | [egressMTLS](#egressmtls) | No | +|``waf`` | The WAF policy configures WAF and log configuration policies for [NGINX AppProtect]({{< ref "/nic/installation/integrations/app-protect-waf/configuration.md" >}}) | [WAF](#waf) | No | +{{% /table %}} + +\* A policy must include exactly one policy. + +### AccessControl + +The access control policy configures NGINX to deny or allow requests from clients with the specified IP addresses/subnets. + +For example, the following policy allows access for clients from the subnet `10.0.0.0/8` and denies access for any other clients: + +```yaml +accessControl: + allow: + - 10.0.0.0/8 +``` + +In contrast, the policy below does the opposite: denies access for clients from `10.0.0.0/8` and allows access for any other clients: + +```yaml +accessControl: + deny: + - 10.0.0.0/8 +``` +{{< note >}} + +The feature is implemented using the NGINX [ngx_http_access_module](http://nginx.org/en/docs/http/ngx_http_access_module.html). NGINX Ingress Controller access control policy supports either allow or deny rules, but not both (as the module does). + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``allow`` | Allows access for the specified networks or addresses. For example, ``192.168.1.1`` or ``10.1.1.0/16``. | ``[]string`` | No | +|``deny`` | Denies access for the specified networks or addresses. For example, ``192.168.1.1`` or ``10.1.1.0/16``. | ``[]string`` | No | \* an accessControl must include either `allow` or `deny`. | +{{% /table %}} + +#### AccessControl Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple access control policies. For example, here we reference two policies, each with configured allow lists: + +```yaml +policies: +- name: allow-policy-one +- name: allow-policy-two +``` + +When you reference more than one access control policy, NGINX Ingress Controller will merge the contents into a single allow list or a single deny list. + +Referencing both allow and deny policies, as shown in the example below, is not supported. If both allow and deny lists are referenced, NGINX Ingress Controller uses just the allow list policies. + +```yaml +policies: +- name: deny-policy +- name: allow-policy-one +- name: allow-policy-two +``` + +### RateLimit + +The rate limit policy configures NGINX to limit the processing rate of requests. + +For example, the following policy will limit all subsequent requests coming from a single IP address once a rate of 10 requests per second is exceeded: + +```yaml +rateLimit: + rate: 10r/s + zoneSize: 10M + key: ${binary_remote_addr} +``` +{{< note >}} + +The feature is implemented using the NGINX [ngx_http_limit_req_module](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html). + +{{< /note >}} + +{{< note >}} + +When the [Zone Sync feature]({{< ref "/nic/configuration/global-configuration/configmap-resource.md#zone-sync" >}}) is enabled with NGINX Plus, the rate limiting zone will be synchronized across all replicas in the cluster. This means all replicas are aware of the requests that have been rate limited by other replicas in the cluster. + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``rate`` | The rate of requests permitted. The rate is specified in requests per second (r/s) or requests per minute (r/m). | ``string`` | Yes | +|``key`` | The key to which the rate limit is applied. Can contain text, variables, or a combination of them. Variables must be surrounded by ``${}``. For example: ``${binary_remote_addr}``. Accepted variables are ``$binary_remote_addr``, ``$request_uri``,``$request_method``, ``$url``, ``$http_``, ``$args``, ``$arg_``, ``$cookie_``, ``$jwt_claim_``. | ``string`` | Yes | +|``zoneSize`` | Size of the shared memory zone. Only positive values are allowed. Allowed suffixes are ``k`` or ``m``, if none are present ``k`` is assumed. | ``string`` | Yes | +|``delay`` | The delay parameter specifies a limit at which excessive requests become delayed. If not set all excessive requests are delayed. | ``int`` | No | +|``noDelay`` | Disables the delaying of excessive requests while requests are being limited. Overrides ``delay`` if both are set. | ``bool`` | No | +|``burst`` | Excessive requests are delayed until their number exceeds the ``burst`` size, in which case the request is terminated with an error. | ``int`` | No | +|``dryRun`` | Enables the dry run mode. In this mode, the rate limit is not actually applied, but the number of excessive requests is accounted as usual in the shared memory zone. | ``bool`` | No | +|``logLevel`` | Sets the desired logging level for cases when the server refuses to process requests due to rate exceeding, or delays request processing. Allowed values are ``info``, ``notice``, ``warn`` or ``error``. Default is ``error``. | ``string`` | No | +|``rejectCode`` | Sets the status code to return in response to rejected requests. Must fall into the range ``400..599``. Default is ``503``. | ``int`` | No | +|``scale`` | Enables a constant rate-limit by dividing the configured rate by the number of nginx-ingress pods currently serving traffic. This adjustment ensures that the rate-limit remains consistent, even as the number of nginx-pods fluctuates due to autoscaling. **This will not work properly if requests from a client are not evenly distributed across all ingress pods** (Such as with sticky sessions, long lived TCP Connections with many requests, and so forth). In such cases using [zone-sync]({{< ref "/nic/configuration/global-configuration/configmap-resource.md#zone-sync" >}}) instead would give better results. Enabling `zone-sync` will suppress this setting. | ``bool`` | No | +|``condition`` | Add a condition to a rate-limit policy. | [ratelimit.condition](#ratelimitcondition) | No | +{{% /table %}} + +{{< note >}} + +For each policy referenced in a VirtualServer and/or its VirtualServerRoutes, NGINX Ingress Controller will generate a single rate limiting zone defined by the [`limit_req_zone`](http://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone) directive. If two VirtualServer resources reference the same policy, NGINX Ingress Controller will generate two different rate limiting zones, one zone per VirtualServer. + +{{< /note >}} + +#### RateLimit Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple rate limit policies. For example, here we reference two policies: + +```yaml +policies: +- name: rate-limit-policy-one +- name: rate-limit-policy-two +``` + +When you reference more than one rate limit policy, NGINX Ingress Controller will configure NGINX to use all referenced rate limits. When you define multiple policies, each additional policy inherits the `dryRun`, `logLevel`, and `rejectCode` parameters from the first policy referenced (`rate-limit-policy-one`, in the example above). + +### RateLimit.Condition + +RateLimit.Condition defines a condition for a rate limit policy. For example: + +```yaml +condition: + jwt: + claim: user_details.level + match: premium + default: true +``` + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``jwt`` | defines a JWT condition to rate limit against. | [ratelimit.condition.jwt](#ratelimitconditionjwt) | No | +|``default`` | sets the rate limit in this policy to be the default if no conditions are met. In a group of policies with the same JWT condition, only one policy can be the default. | ``bool`` | No | +{{% /table %}} + +The rate limit policy with condition is designed to be used in combination with one or more rate limit policies. For example, multiple rate limit policies with [RateLimit.Condition.JWT](#ratelimitconditionjwt) can be used to apply different tiers of rate limit based on the value of a JWT claim. For a practical example of tiered rate limiting by the value of a JWT claim, see the example in our [GitHub repository](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/rate-limit-tiered-jwt-claim/README.md). + +### RateLimit.Condition.JWT +{{< note >}} + +This feature is only available with NGINX Plus. + +{{< /note >}} + +RateLimit.Condition.JWT defines a condition for a rate limit by JWT claim. For example, here we define a condition for a rate limit policy that only applies to requests with a JWT claim `user_details.level` with a value `premium`: + +```yaml +jwt: + claim: user_details.level + match: premium +``` + +The rate limit policy will only apply to requests that contain a JWT with the specified claim and value. For example, the following JWT payload will match the JWT condition: + +```json +{ + "user_details": { + "level": "premium" + }, + "sub": "client1" +} +``` + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``claim`` | Claim is the JWT claim to be rate limit by. Nested claims should be separated by ".". | ``string`` | Yes | +|``match`` | the value of the claim to match against. | ``string`` | Yes | +{{% /table %}} + +### APIKey + +The API Key auth policy configures NGINX to authorize client requests based on the presence of a valid API Key in a header or query param specified in the policy. + +{{< note >}} + +The feature is implemented using NGINX [ngx_http_auth_request_module](http://nginx.org/en/docs/http/ngx_http_auth_request_module.html) and [NGINX JavaScript (NJS)](https://nginx.org/en/docs/njs/). + +{{< /note >}} + +The policies' API keys are securely stored using SHA-256 hashing. When a client sends an API Key, it is hashed by NJS and then compared to the hashed API Key in the NGINX config. + +If the hashed keys match, the NGINX JavaScript (NJS) subrequest issues a 204 No Content response to the `auth_request` directive, indicating successful authorization. Conversely, if no API Key is provided in the specified header or query parameter, a 401 Unauthorized response is returned. Similarly, if an invalid key is presented in the expected header or query parameter, a 403 Forbidden response is issued, denying access. + +It is possible to use the [errorPages](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#errorpage) property on a route, to change the default behaviour of 401 or 403 errors. + +At least one header or query param is required. + +The policy below configures NGINX Ingress Controller to require the API Key `password` in the header "my-header". + +```yaml +apiKey: + suppliedIn: + header: + - "my-header" + clientSecret: api-key-secret +``` + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: api-key-secret +type: nginx.org/apikey +data: + client1: cGFzc3dvcmQ= # password +``` + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``suppliedIn`` | `header` or `query`. | | Yes | +|``suppliedIn.header`` | An array of headers that the API Key may appear in. | ``string[]`` | No | +|``suppliedIn.query`` | An array of query params that the API Key may appear in. | ``string[]`` | No | +|``clientSecret`` | The name of the Kubernetes secret that stores the API Key(s). It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/apikey``, and the API Key(s) must be stored in a key: val format where each key is a unique clientID and each value is a unique base64 encoded API Key | ``string`` | Yes | +{{% /table %}} + +{{}}An APIKey Policy must include a minimum of one of the `suppliedIn.header` or `suppliedIn.query` parameters. Both can also be supplied.{{}} + +#### APIKey Merging Behavior + +A VirtualServer or VirtualServerRoute can be associated with only one API Key policy per route or subroute. However, it is possible to replace an API Key policy from a higher-level with a different policy defined on a more specific route. + +For example, a VirtualServer can implement different API Key policies at various levels. In the configuration below, the server-wide api-key-policy-server applies to /backend1 for authorization, as it lacks a more specific policy. Meanwhile, /backend2 uses the api-key-policy-route defined at the route level. + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server +spec: + host: virtual-server.example.com + policies: + - name: api-key-policy-server + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1 + service: backend1-svc + port: 80 + routes: + - path: /backend1 + action: + pass: backend1 + - path: /backend2 + action: + pass: backend2 + policies: + - name: api-key-policy-route +``` + +### BasicAuth + +The basic auth policy configures NGINX to authenticate client requests using the [HTTP Basic authentication scheme](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication). + +For example, the following policy will reject all requests that do not include a valid username/password combination in the HTTP header `Authentication` + +```yaml +basicAuth: + secret: htpasswd-secret + realm: "My API" +``` +{{< note >}} +The feature is implemented using the NGINX [ngx_http_auth_basic_module](https://nginx.org/en/docs/http/ngx_http_auth_basic_module.html). +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``secret`` | The name of the Kubernetes secret that stores the Htpasswd configuration. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/htpasswd``, and the config must be stored in the secret under the key ``htpasswd``, otherwise the secret will be rejected as invalid. | ``string`` | Yes | +|``realm`` | The realm for the basic authentication. | ``string`` | No | +{{% /table %}} + +#### BasicAuth Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple basic auth policies. However, only one can be applied. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: basic-auth-policy-one +- name: basic-auth-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `basic-auth-policy-one`, and ignores `basic-auth-policy-two`. + +### JWT Using Local Kubernetes Secret + +{{< note >}} + +This feature is only available with NGINX Plus. + +{{< /note >}} + +The JWT policy configures NGINX Plus to authenticate client requests using JSON Web Tokens. + +The following example policy will reject all requests that do not include a valid JWT in the HTTP header `token`: + +```yaml +jwt: + secret: jwk-secret + realm: "My API" + token: $http_token +``` + +You can pass the JWT claims and JOSE headers to the upstream servers. For example: + +```yaml +action: + proxy: + upstream: webapp + requestHeaders: + set: + - name: user + value: ${jwt_claim_user} + - name: alg + value: ${jwt_header_alg} +``` + +We use the `requestHeaders` of the [Action.Proxy](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#actionproxy) to set the values of two headers that NGINX will pass to the upstream servers. + +The value of the `${jwt_claim_user}` variable is the `user` claim of a JWT. For other claims, use `${jwt_claim_name}`, where `name` is the name of the claim. Note that nested claims and claims that include a period (`.`) are not supported. Similarly, use `${jwt_header_name}` where `name` is the name of a header. In our example, we use the `alg` header. + +{{< note >}} + +This feature is implemented using the NGINX Plus [ngx_http_auth_jwt_module](https://nginx.org/en/docs/http/ngx_http_auth_jwt_module.html). + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``secret`` | The name of the Kubernetes secret that stores the JWK. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/jwk``, and the JWK must be stored in the secret under the key ``jwk``, otherwise the secret will be rejected as invalid. | ``string`` | Yes | +|``realm`` | The realm of the JWT. | ``string`` | Yes | +|``token`` | The token specifies a variable that contains the JSON Web Token. By default the JWT is passed in the ``Authorization`` header as a Bearer Token. JWT may be also passed as a cookie or a part of a query string, for example: ``$cookie_auth_token``. Accepted variables are ``$http_``, ``$arg_``, ``$cookie_``. | ``string`` | No | +{{% /table %}} + +#### JWT Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple JWT policies. However, only one can be applied: every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: jwt-policy-one +- name: jwt-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `jwt-policy-one`, and ignores `jwt-policy-two`. + +### JWT Using JWKS From Remote Location + +{{< note >}} + +This feature is only available with NGINX Plus. + +{{< /note >}} + +The JWT policy configures NGINX Plus to authenticate client requests using JSON Web Tokens, allowing import of the keys (JWKS) for JWT policy by means of a URL (for a remote server or an identity provider) as a result they don't have to be copied and updated to the IC pod. + +The following example policy will reject all requests that do not include a valid JWT in the HTTP header fetched from the identity provider: + +```yaml +jwt: + realm: MyProductAPI + token: $http_token + jwksURI: + keyCache: 1h +``` + +{{< note >}} + +This feature is implemented using the NGINX Plus directive [auth_jwt_key_request](http://nginx.org/en/docs/http/ngx_http_auth_jwt_module.html#auth_jwt_key_request) under [ngx_http_auth_jwt_module](https://nginx.org/en/docs/http/ngx_http_auth_jwt_module.html). + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``jwksURI`` | The remote URI where the request will be sent to retrieve JSON Web Key set| ``string`` | Yes | +|``keyCache`` | Enables in-memory caching of JWKS (JSON Web Key Sets) that are obtained from the ``jwksURI`` and sets a valid time for expiration. | ``string`` | Yes | +|``realm`` | The realm of the JWT. | ``string`` | Yes | +|``token`` | The token specifies a variable that contains the JSON Web Token. By default the JWT is passed in the ``Authorization`` header as a Bearer Token. JWT may be also passed as a cookie or a part of a query string, for example: ``$cookie_auth_token``. Accepted variables are ``$http_``, ``$arg_``, ``$cookie_``. | ``string`` | No | +{{% /table %}} + +{{< note >}} + +Content caching is enabled by default for each JWT policy with a default time of 12 hours. + +This is done to ensure to improve resiliency by allowing the JWKS (JSON Web Key Set) to be retrieved from the cache even when it has expired. + +{{< /note >}} + +#### JWT Merging Behavior + +This behavior is similar to using a local Kubernetes secret where a VirtualServer/VirtualServerRoute can reference multiple JWT policies. However, only one can be applied: every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: jwt-policy-one +- name: jwt-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `jwt-policy-one`, and ignores `jwt-policy-two`. + +### IngressMTLS + +The IngressMTLS policy configures client certificate verification. + +For example, the following policy will verify a client certificate using the CA certificate specified in the `ingress-mtls-secret`: + +```yaml +ingressMTLS: + clientCertSecret: ingress-mtls-secret + verifyClient: "on" + verifyDepth: 1 +``` + +Below is an example of the `ingress-mtls-secret` using the secret type `nginx.org/ca` + +```yaml +kind: Secret +metadata: + name: ingress-mtls-secret +apiVersion: v1 +type: nginx.org/ca +data: + ca.crt: +``` + +A VirtualServer that references an IngressMTLS policy must: + +- Enable [TLS termination](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#virtualservertls). +- Reference the policy in the VirtualServer [`spec`](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#virtualserver-specification). It is not allowed to reference an IngressMTLS policy in a [`route`](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#virtualserverroute) or in a VirtualServerRoute [`subroute`](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#virtualserverroutesubroute). + +If the conditions above are not met, NGINX will send the `500` status code to clients. + +You can pass the client certificate details, including the certificate, to the upstream servers. For example: + +```yaml +action: + proxy: + upstream: webapp + requestHeaders: + set: + - name: client-cert-subj-dn + value: ${ssl_client_s_dn} # subject DN + - name: client-cert + value: ${ssl_client_escaped_cert} # client certificate in the PEM format (urlencoded) +``` + +We use the `requestHeaders` of the [Action.Proxy](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#actionproxy) to set the values of the two headers that NGINX will pass to the upstream servers. See the [list of embedded variables](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#variables) that are supported by the `ngx_http_ssl_module`, which you can use to pass the client certificate details. + +{{< note >}} + + The feature is implemented using the NGINX [ngx_http_ssl_module](https://nginx.org/en/docs/http/ngx_http_ssl_module.html). + + {{< /note >}} + +#### Using a Certificate Revocation List + +The IngressMTLS policy supports configuring at CRL for your policy. +This can be done in one of two ways. + +{{< note >}} + + Only one of these configurations options can be used at a time. + +{{< /note >}} + +1. Adding the `ca.crl` field to the `nginx.org/ca` secret type, which accepts a base64 encoded certificate revocation list (crl). + Example YAML: + +```yaml +kind: Secret +metadata: + name: ingress-mtls-secret +apiVersion: v1 +type: nginx.org/ca +data: + ca.crt: + ca.crl: +``` + +2. Adding the `crlFileName` field to your IngressMTLS policy spec with the name of the CRL file. + +{{< note >}} + +This configuration option should only be used when using a CRL that is larger than 1MiB. + +Otherwise we recommend using the `nginx.org/ca` secret type for managing your CRL. + +{{< /note >}} + +Example YAML: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: Policy +metadata: + name: ingress-mtls-policy +spec: +ingressMTLS: + clientCertSecret: ingress-mtls-secret + crlFileName: webapp.crl + verifyClient: "on" + verifyDepth: 1 +``` + +**IMPORTANT NOTE** +When configuring a CRL with the `ingressMTLS.crlFileName` field, there is additional context to keep in mind: + +1. NGINX Ingress Controller will expect the CRL, in this case `webapp.crl`, will be in `/etc/nginx/secrets`. A volume mount will need to be added to NGINX Ingress Controller deployment add your CRL to `/etc/nginx/secrets` +2. When updating the content of your CRL (e.g a new certificate has been revoked), NGINX will need to be reloaded to pick up the latest changes. Depending on your environment this may require updating the name of your CRL and applying this update to your `ingress-mtls.yaml` policy to ensure NGINX picks up the latest CRL. + +Please refer to the Kubernetes documentation on [volumes](https://kubernetes.io/docs/concepts/storage/volumes/) to find the best implementation for your environment. + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``clientCertSecret`` | The name of the Kubernetes secret that stores the CA certificate. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/ca``, and the certificate must be stored in the secret under the key ``ca.crt``, otherwise the secret will be rejected as invalid. | ``string`` | Yes | +|``verifyClient`` | Verification for the client. Possible values are ``"on"``, ``"off"``, ``"optional"``, ``"optional_no_ca"``. The default is ``"on"``. | ``string`` | No | +|``verifyDepth`` | Sets the verification depth in the client certificates chain. The default is ``1``. | ``int`` | No | +|``crlFileName`` | The file name of the Certificate Revocation List. NGINX Ingress Controller will look for this file in `/etc/nginx/secrets` | ``string`` | No | +{{% /table %}} + +#### IngressMTLS Merging Behavior + +A VirtualServer can reference only a single IngressMTLS policy. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: ingress-mtls-policy-one +- name: ingress-mtls-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `ingress-mtls-policy-one`, and ignores `ingress-mtls-policy-two`. + +### EgressMTLS + +The EgressMTLS policy configures upstreams authentication and certificate verification. + +For example, the following policy will use `egress-mtls-secret` to authenticate with the upstream application and `egress-trusted-ca-secret` to verify the certificate of the application: + +```yaml +egressMTLS: + tlsSecret: egress-mtls-secret + trustedCertSecret: egress-trusted-ca-secret + verifyServer: on + verifyDepth: 2 +``` + +{{< note >}} + +The feature is implemented using the NGINX [ngx_http_proxy_module](https://nginx.org/en/docs/http/ngx_http_proxy_module.html). + +{{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``tlsSecret`` | The name of the Kubernetes secret that stores the TLS certificate and key. It must be in the same namespace as the Policy resource. The secret must be of the type ``kubernetes.io/tls``, the certificate must be stored in the secret under the key ``tls.crt``, and the key must be stored under the key ``tls.key``, otherwise the secret will be rejected as invalid. | ``string`` | No | +|``trustedCertSecret`` | The name of the Kubernetes secret that stores the CA certificate. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/ca``, and the certificate must be stored in the secret under the key ``ca.crt``, otherwise the secret will be rejected as invalid. | ``string`` | No | +|``verifyServer`` | Enables verification of the upstream HTTPS server certificate. | ``bool`` | No | +|``verifyDepth`` | Sets the verification depth in the proxied HTTPS server certificates chain. The default is ``1``. | ``int`` | No | +|``sessionReuse`` | Enables reuse of SSL sessions to the upstreams. The default is ``true``. | ``bool`` | No | +|``serverName`` | Enables passing of the server name through ``Server Name Indication`` extension. | ``bool`` | No | +|``sslName`` | Allows overriding the server name used to verify the certificate of the upstream HTTPS server. | ``string`` | No | +|``ciphers`` | Specifies the enabled ciphers for requests to an upstream HTTPS server. The default is ``DEFAULT``. | ``string`` | No | +|``protocols`` | Specifies the protocols for requests to an upstream HTTPS server. The default is ``TLSv1 TLSv1.1 TLSv1.2``. | ``string`` | No | > Note: the value of ``ciphers`` and ``protocols`` is not validated by NGINX Ingress Controller. As a result, NGINX can fail to reload the configuration. To ensure that the configuration for a VirtualServer/VirtualServerRoute that references the policy was successfully applied, check its [status](/nginx-ingress-controller/configuration/global-configuration/reporting-resources-status/#virtualserver-and-virtualserverroute-resources). The validation will be added in the future releases. | +{{% /table %}} + +#### EgressMTLS Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple EgressMTLS policies. However, only one can be applied. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: egress-mtls-policy-one +- name: egress-mtls-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `egress-mtls-policy-one`, and ignores `egress-mtls-policy-two`. + +### OIDC + +{{< tip >}} + +This feature is disabled by default. To enable it, set the [enable-oidc]({{< ref "/nic/configuration/global-configuration/command-line-arguments.md#cmdoption-enable-oidc" >}}) command-line argument of NGINX Ingress Controller. + +{{< /tip >}} + +The OIDC policy configures NGINX Plus as a relying party for OpenID Connect authentication. + +For example, the following policy will use the client ID `nginx-plus` and the client secret `oidc-secret` to authenticate with the OpenID Connect provider `https://idp.example.com`: + +```yaml +spec: + oidc: + clientID: nginx-plus + clientSecret: oidc-secret + authEndpoint: https://idp.example.com/openid-connect/auth + tokenEndpoint: https://idp.example.com/openid-connect/token + jwksURI: https://idp.example.com/openid-connect/certs + endSessionEndpoint: https://idp.example.com/openid-connect/logout + postLogoutRedirectURI: / + accessTokenEnable: true + pkceEnable: false +``` + +NGINX Plus will pass the ID of an authenticated user to the backend in the HTTP header `username`. + +{{< note >}} + +The feature is implemented using the [reference implementation](https://github.com/nginxinc/nginx-openid-connect/) of NGINX Plus as a relying party for OpenID Connect authentication. + +{{< /note >}} + +#### Prerequisites + +In order to use OIDC, you need to enable [zone synchronization](https://docs.nginx.com/nginx/admin-guide/high-availability/zone_sync/). If you don't set up zone synchronization, NGINX Plus will fail to reload. +You also need to configure a resolver, which NGINX Plus will use to resolve the IDP authorization endpoint. You can find an example configuration [in our GitHub repository](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/examples/custom-resources/oidc#step-7---configure-nginx-plus-zone-synchronization-and-resolver). + +{{< warning >}} + +The configuration in the example doesn't enable TLS and the synchronization between the replica happens in clear text. This could lead to the exposure of tokens. + +{{< /warning >}} + +#### Limitations + +The OIDC policy defines a few internal locations that can't be customized: `/_jwks_uri`, `/_token`, `/_refresh`, `/_id_token_validation`, `/logout`. In addition, as explained below, `/_codexch` is the default value for redirect URI, and `/_logout` is the default value for post logout redirect URI, both of which can be customized. Specifying one of these locations as a route in the VirtualServer or VirtualServerRoute will result in a collision and NGINX Plus will fail to reload. + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``clientID`` | The client ID provided by your OpenID Connect provider. | ``string`` | Yes | +|``clientSecret`` | The name of the Kubernetes secret that stores the client secret provided by your OpenID Connect provider. It must be in the same namespace as the Policy resource. The secret must be of the type ``nginx.org/oidc``, and the secret under the key ``client-secret``, otherwise the secret will be rejected as invalid. If PKCE is enabled, this should be not configured. | ``string`` | Yes | +|``authEndpoint`` | URL for the authorization endpoint provided by your OpenID Connect provider. | ``string`` | Yes | +|``authExtraArgs`` | A list of extra URL arguments to pass to the authorization endpoint provided by your OpenID Connect provider. Arguments must be URL encoded, multiple arguments may be included in the list, for example ``[ arg1=value1, arg2=value2 ]`` | ``string[]`` | No | +|``tokenEndpoint`` | URL for the token endpoint provided by your OpenID Connect provider. | ``string`` | Yes | +|``endSessionEndpoint`` | URL provided by your OpenID Connect provider to request the end user be logged out. | ``string`` | No | +|``jwksURI`` | URL for the JSON Web Key Set (JWK) document provided by your OpenID Connect provider. | ``string`` | Yes | +|``scope`` | List of OpenID Connect scopes. The scope ``openid`` always needs to be present and others can be added concatenating them with a ``+`` sign, for example ``openid+profile+email``, ``openid+email+userDefinedScope``. The default is ``openid``. | ``string`` | No | +|``redirectURI`` | Allows overriding the default redirect URI. The default is ``/_codexch``. | ``string`` | No | +|``postLogoutRedirectURI`` | URI to redirect to after the logout has been performed. Requires ``endSessionEndpoint``. The default is ``/_logout``. | ``string`` | No | +|``zoneSyncLeeway`` | Specifies the maximum timeout in milliseconds for synchronizing ID/access tokens and shared values between Ingress Controller pods. The default is ``200``. | ``int`` | No | +|``accessTokenEnable`` | Option of whether Bearer token is used to authorize NGINX to access protected backend. | ``boolean`` | No | +|``pkceEnable`` | Switches Proof Key for Code Exchange on. The OpenID client needs to be in public mode. `clientSecret` is not used in this mode. | ``boolean`` | No | +{{% /table %}} + +{{< note >}} + +Only one OIDC policy can be referenced in a VirtualServer and its VirtualServerRoutes. However, the same policy can still be applied to different routes in the VirtualServer and VirtualServerRoutes. + +{{< /note >}} + +#### OIDC Merging Behavior + +A VirtualServer/VirtualServerRoute can reference only a single OIDC policy. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: oidc-policy-one +- name: oidc-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `oidc-policy-one`, and ignores `oidc-policy-two`. + +## Using Policy + +You can use the usual `kubectl` commands to work with Policy resources, just as with built-in Kubernetes resources. + +For example, the following command creates a Policy resource defined in `access-control-policy-allow.yaml` with the name `webapp-policy`: + +```shell +kubectl apply -f access-control-policy-allow.yaml + +policy.k8s.nginx.org/webapp-policy configured +``` + +You can get the resource by running: + +```shell +kubectl get policy webapp-policy + +NAME AGE +webapp-policy 27m +``` + +For `kubectl get` and similar commands, you can also use the short name `pol` instead of `policy`. + +### WAF {#waf} + +{{< note >}} The feature is implemented using the NGINX Plus [NGINX App Protect WAF Module](https://docs.nginx.com/nginx-app-protect/configuration/). {{< /note >}} + +The WAF policy configures NGINX Plus to secure client requests using App Protect WAF policies. + +For example, the following policy will enable the referenced APPolicy. You can configure multiple APLogConfs with log destinations: + +```yaml +waf: + enable: true + apPolicy: "default/dataguard-alarm" + securityLogs: + - enable: true + apLogConf: "default/logconf" + logDest: "syslog:server=syslog-svc.default:514" + - enable: true + apLogConf: "default/logconf" + logDest: "syslog:server=syslog-svc-secondary.default:514" +``` + +{{< note >}} The field `waf.securityLog` is deprecated and will be removed in future releases.It will be ignored if `waf.securityLogs` is populated. {{< /note >}} + +{{% table %}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables NGINX App Protect WAF. | ``bool`` | Yes | +|``apPolicy`` | The [App Protect WAF policy]({{< ref "/nic/installation/integrations/app-protect-waf/configuration.md#waf-policies" >}}) of the WAF. Accepts an optional namespace. Mutually exclusive with ``apBundle``. | ``string`` | No | +|``apBundle`` | The [App Protect WAF policy bundle]({{< ref "/nic/installation/integrations/app-protect-waf/configuration.md#waf-bundles" >}}). Mutually exclusive with ``apPolicy``. | ``string`` | No | +|``securityLog.enable`` | Enables security log. | ``bool`` | No | +|``securityLog.apLogConf`` | The [App Protect WAF log conf]({{< ref "/nic/installation/integrations/app-protect-waf/configuration.md#waf-logs" >}}) resource. Accepts an optional namespace. Only works with ``apPolicy``. | ``string`` | No | +|``securityLog.apLogBundle`` | The [App Protect WAF log bundle]({{< ref "/nic/installation/integrations/app-protect-waf/configuration.md#waf-bundles" >}}) resource. Only works with ``apBundle``. | ``string`` | No | +|``securityLog.logDest`` | The log destination for the security log. Only accepted variables are ``syslog:server=:``, ``stderr``, ````. | ``string`` | No | +{{% /table %}} + +#### WAF Merging Behavior + +A VirtualServer/VirtualServerRoute can reference multiple WAF policies. However, only one can be applied. Every subsequent reference will be ignored. For example, here we reference two policies: + +```yaml +policies: +- name: waf-policy-one +- name: waf-policy-two +``` + +In this example NGINX Ingress Controller will use the configuration from the first policy reference `waf-policy-one`, and ignores `waf-policy-two`. + +### Applying Policies + +You can apply policies to both VirtualServer and VirtualServerRoute resources. For example: + +- VirtualServer: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: VirtualServer + metadata: + name: cafe + namespace: cafe + spec: + host: cafe.example.com + tls: + secret: cafe-secret + policies: # spec policies + - name: policy1 + upstreams: + - name: coffee + service: coffee-svc + port: 80 + routes: + - path: /tea + policies: # route policies + - name: policy2 + namespace: cafe + route: tea/tea + - path: /coffee + policies: # route policies + - name: policy3 + namespace: cafe + action: + pass: coffee + ``` + + For VirtualServer, you can apply a policy: + * to all routes (spec policies) + * to a specific route (route policies) + + Route policies of the *same type* override spec policies. In the example above, if the type of the policies `policy-1` and `policy-3` is `accessControl`, then for requests to `cafe.example.com/coffee`, NGINX will apply `policy-3`. + + The overriding is enforced by NGINX: the spec policies are implemented in the `server` context of the config, and the route policies are implemented in the `location` context. As a result, the route policies of the same type win. + +- VirtualServerRoute, which is referenced by the VirtualServer above: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: VirtualServerRoute + metadata: + name: tea + namespace: tea + spec: + host: cafe.example.com + upstreams: + - name: tea + service: tea-svc + port: 80 + subroutes: # subroute policies + - path: /tea + policies: + - name: policy4 + namespace: tea + action: + pass: tea + ``` + + For VirtualServerRoute, you can apply a policy to a subroute (subroute policies). + + Subroute policies of the same type override spec policies. In the example above, if the type of the policies `policy-1` (in the VirtualServer) and `policy-4` is `accessControl`, then for requests to `cafe.example.com/tea`, NGINX will apply `policy-4`. As with the VirtualServer, the overriding is enforced by NGINX. + + Subroute policies always override route policies no matter the types. For example, the policy `policy-2` in the VirtualServer route will be ignored for the subroute `/tea`, because the subroute has its own policies (in our case, only one policy `policy4`). If the subroute didn't have any policies, then the `policy-2` would be applied. This overriding is enforced by NGINX Ingress Controller -- the `location` context for the subroute will either have route policies or subroute policies, but not both. + +### Invalid Policies + +NGINX will treat a policy as invalid if one of the following conditions is met: + +- The policy doesn't pass the [comprehensive validation](#comprehensive-validation). +- The policy isn't present in the cluster. +- The policy doesn't meet its type-specific requirements. For example, an `ingressMTLS` policy requires TLS termination enabled in the VirtualServer. + +For an invalid policy, NGINX returns the 500 status code for client requests with the following rules: + +- If a policy is referenced in a VirtualServer `route` or a VirtualServerRoute `subroute`, then NGINX will return the 500 status code for requests for the URIs of that route/subroute. +- If a policy is referenced in the VirtualServer `spec`, then NGINX will return the 500 status code for requests for all URIs of that VirtualServer. + +If a policy is invalid, the VirtualServer or VirtualServerRoute will have the [status](/nginx-ingress-controller/configuration/global-configuration/reporting-resources-status#virtualserver-and-virtualserverroute-resources) with the state `Warning` and the message explaining why the policy wasn't considered invalid. + +### Validation + +Two types of validation are available for the Policy resource: + +- *Structural validation*, done by `kubectl` and the Kubernetes API server. +- *Comprehensive validation*, done by NGINX Ingress Controller. + +#### Structural Validation + +The custom resource definition for the Policy includes a structural OpenAPI schema, which describes the type of every field of the resource. + +If you try to create (or update) a resource that violates the structural schema -- for example, the resource uses a string value instead of an array of strings in the `allow` field -- `kubectl` and the Kubernetes API server will reject the resource. + +- Example of `kubectl` validation: + + ```shell + kubectl apply -f access-control-policy-allow.yaml + + error: error validating "access-control-policy-allow.yaml": error validating data: ValidationError(Policy.spec.accessControl.allow): invalid type for org.nginx.k8s.v1.Policy.spec.accessControl.allow: got "string", expected "array"; if you choose to ignore these errors, turn validation off with --validate=false + ``` + +- Example of Kubernetes API server validation: + + ```shell + kubectl apply -f access-control-policy-allow.yaml --validate=false + + The Policy "webapp-policy" is invalid: spec.accessControl.allow: Invalid value: "string": spec.accessControl.allow in body must be of type array: "string" + ``` + +If a resource passes structural validation, then NGINX Ingress Controller's comprehensive validation runs. + +#### Comprehensive Validation + +NGINX Ingress Controller validates the fields of a Policy resource. If a resource is invalid, NGINX Ingress Controller will reject it. The resource will continue to exist in the cluster, but NGINX Ingress Controller will ignore it. + +You can use `kubectl` to check whether or not NGINX Ingress Controller successfully applied a Policy configuration. For our example `webapp-policy` Policy, we can run: + +```shell +kubectl describe pol webapp-policy + +. . . +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 11s nginx-ingress-controller Policy default/webapp-policy was added or updated +``` + +Note how the events section includes a Normal event with the AddedOrUpdated reason that informs us that the configuration was successfully applied. + +If you create an invalid resource, NGINX Ingress Controller will reject it and emit a Rejected event. For example, if you create a Policy `webapp-policy` with an invalid IP `10.0.0.` in the `allow` field, you will get: + +```shell +kubectl describe policy webapp-policy + +. . . +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 7s nginx-ingress-controller Policy default/webapp-policy is invalid and was rejected: spec.accessControl.allow[0]: Invalid value: "10.0.0.": must be a CIDR or IP +``` + +Note how the events section includes a Warning event with the Rejected reason. + +Additionally, this information is also available in the `status` field of the Policy resource. Note the Status section of the Policy: + +```shell +kubectl describe pol webapp-policy + +. . . +Status: + Message: Policy default/webapp-policy is invalid and was rejected: spec.accessControl.allow[0]: Invalid value: "10.0.0.": must be a CIDR or IP + Reason: Rejected + State: Invalid +``` + +**Note**: If you make an existing resource invalid, NGINX Ingress Controller will reject it. diff --git a/content/nic/configuration/security.md b/content/nic/configuration/security.md new file mode 100644 index 000000000..f0b74f6a3 --- /dev/null +++ b/content/nic/configuration/security.md @@ -0,0 +1,102 @@ +--- +docs: DOCS-597 +doctypes: +- '' +title: Security recommendations +toc: true +weight: 300 +--- + +F5 NGINX Ingress Controller follows Kubernetes best practices: this page outlines configuration specific to NGINX Ingress Controller you may require, including links to examples in the [GitHub repository](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples). + +For general guidance, we recommend the official Kubernetes documentation for [Securing a Cluster](https://kubernetes.io/docs/tasks/administer-cluster/securing-a-cluster/). + +## Kubernetes recommendations + +### RBAC and Service Accounts + +Kubernetes uses [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) to control the resources and operations available to different types of users. + +NGINX Ingress Controller requires RBAC to configure a [ServiceUser](https://kubernetes.io/docs/concepts/security/service-accounts/#default-service-accounts), and provides least privilege access in its standard deployment configurations: + +- [Helm](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/deployments/rbac/rbac.yaml) +- [Manifests](https://github.com/nginx/kubernetes-ingress/blob/v{{< nic-version >}}/deployments/rbac/rbac.yaml) + +By default, the ServiceAccount has access to all Secret resources in the cluster. + +### Secrets + +[Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) are required by NGINX Ingress Controller for certificates and privacy keys, which Kubernetes stores unencrypted by default. We recommend following the [Kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) to store these Secrets using at-rest encryption. + + +## NGINX Ingress Controller recommendations + +### Configure root filesystem as read-only + +{{< caution >}} + This feature is compatible with [NGINX App Protect WAFv5](https://docs.nginx.com/nginx-app-protect-waf/v5/). It is not compatible with [NGINX App Protect WAFv4](https://docs.nginx.com/nginx-app-protect-waf/v4/) or [NGINX App Protect DoS](https://docs.nginx.com/nginx-app-protect-dos/). +{{< /caution >}} + +NGINX Ingress Controller is designed to be resilient against attacks in various ways, such as running the service as non-root to avoid changes to files. We recommend setting filesystems on all containers to read-only, this includes `nginx-ingress-controller`, though also includes `waf-enforcer` and `waf-config-mgr` when NGINX App Protect WAFv5 is in use. This is so that the attack surface is further reduced by limiting changes to binaries and libraries. + +This is not enabled by default, but can be enabled with **Helm** using the [**readOnlyRootFilesystem**]({{< ref "/nic/installation/installing-nic/installation-with-helm.md#configuration" >}}) argument in security contexts on all containers: `nginx-ingress-controller`, `waf_enforcer` and `waf_config_mgr`. + +For **Manifests**, uncomment the following sections of the deployment and add sections for `waf-enforcer` and `waf-config-mgr` containers: + +- `readOnlyRootFilesystem: true` +- The entire **volumeMounts** section +- The entire **initContainers** section + +The block below shows the code you will look for: + +```yaml +# volumes: +# - name: nginx-etc +# emptyDir: {} +# - name: nginx-cache +# emptyDir: {} +# - name: nginx-lib +# emptyDir: {} +# - name: nginx-log +# emptyDir: {} +. +. +. +# readOnlyRootFilesystem: true +. +. +. +# volumeMounts: +# - mountPath: /etc/nginx +# name: nginx-etc +# - mountPath: /var/cache/nginx +# name: nginx-cache +# - mountPath: /var/lib/nginx +# name: nginx-lib +# - mountPath: /var/log/nginx +# name: nginx-log +``` + +- Add **waf-enforcer** and **waf-config-mgr** container sections +- Add `readOnlyFilesystem: true` in both containers security context sections + +### Prometheus + +If Prometheus metrics are [enabled]({{< ref "/nic/logging-and-monitoring/prometheus.md" >}}), we recommend [using HTTPS]({{< ref "/nic/configuration/global-configuration/command-line-arguments.md#cmdoption-prometheus-tls-secret" >}}). + +### Snippets + +Snippets allow raw NGINX configuration to be inserted into resources. They are intended for advanced NGINX users and could create vulnerabilities in a cluster if misused. + +Snippets are disabled by default. To use snippets, set the [**enable-snippets**]({{< ref"/nic/configuration/global-configuration/command-line-arguments.md#cmdoption-enable-snippets" >}}) command-line argument. + +{{< caution >}} + Snippets are **always** enabled for ConfigMap. +{{< /caution >}} + +For more information, read the following: + +- [Advanced configuration using Snippets]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md" >}}) +- [Using Snippets with VirtualServer/VirtualServerRoute]({{< ref "/nic/configuration/virtualserver-and-virtualserverroute-resources.md#using-snippets" >}}) +- [Using Snippets with TransportServer]({{< ref "/nic/configuration/transportserver-resource.md#using-snippets" >}}) +- [ConfigMap snippets and custom templates]({{< ref "/nic/configuration/global-configuration/configmap-resource.md#snippets-and-custom-templates" >}}) diff --git a/content/nic/configuration/transportserver-resource.md b/content/nic/configuration/transportserver-resource.md new file mode 100644 index 000000000..b725ea2e6 --- /dev/null +++ b/content/nic/configuration/transportserver-resource.md @@ -0,0 +1,416 @@ +--- +title: TransportServer resources +toc: true +weight: 600 +docs: DOCS-598 +--- + +This document is reference material for the TransportServer resource used by F5 NGINX Ingress Controller. + +The TransportServer resource allows you to configure TCP, UDP, and TLS Passthrough load balancing. The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + +The GitHub repository has [examples of the resources](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources) for specific use cases. + +## Prerequisites + +- For TCP and UDP, the TransportServer resource must be used in conjunction with the [GlobalConfiguration resource]({{< ref "/nic/configuration/global-configuration/globalconfiguration-resource.md" >}}), which must be created separately. +- For TLS Passthrough, make sure to enable the [`-enable-tls-passthrough`]({{< ref "/nic/configuration/global-configuration/command-line-arguments#cmdoption-enable-tls-passthrough.md" >}}) command-line argument of NGINX Ingress Controller. + +## TransportServer Specification + +The TransportServer resource defines load balancing configuration for TCP, UDP, or TLS Passthrough traffic. Below are a few examples: + +- TCP load balancing: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: dns-tcp + spec: + host: dns.example.com + listener: + name: dns-tcp + protocol: TCP + tls: + secret: cafe-secret + upstreams: + - name: dns-app + service: dns-service + port: 5353 + action: + pass: dns-app + ``` + +- UDP load balancing: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: dns-udp + spec: + listener: + name: dns-udp + protocol: UDP + upstreams: + - name: dns-app + service: dns-service + port: 5353 + upstreamParameters: + udpRequests: 1 + udpResponses: 1 + action: + pass: dns-app + ``` + +- TLS passthrough load balancing: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: TransportServer + metadata: + name: secure-app + spec: + listener: + name: tls-passthrough + protocol: TLS_PASSTHROUGH + host: app.example.com + upstreams: + - name: secure-app + service: secure-app + port: 8443 + action: + pass: secure-app + ``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``listener`` | The listener on NGINX that will accept incoming connections/datagrams. | [listener](#listener) | Yes | +|``host`` | The host (domain name) of the server. Must be a valid subdomain as defined in RFC 1123, such as ``my-app`` or ``hello.example.com``. Wildcard domains like ``*.example.com`` are not allowed. When specified, NGINX will use this host for SNI-based routing. For TLS Passthrough, this field is required. For TCP with TLS termination, specifying the host enables SNI routing and requires specifying a TLS secret.| ``string`` | No | +|``tls`` | The TLS termination configuration. Not supported for TLS Passthrough load balancing. | [tls](#tls) | No | +|``upstreams`` | A list of upstreams. | [[]upstream](#upstream) | Yes | +|``upstreamParameters`` | The upstream parameters. | [upstreamParameters](#upstreamparameters) | No | +|``action`` | The action to perform for a client connection/datagram. | [action](#action) | Yes | +|``ingressClassName`` | Specifies which Ingress Controller must handle the TransportServer resource. | ``string`` | No | +|``streamSnippets`` | Sets a custom snippet in the ``stream`` context. | ``string`` | No | +|``serverSnippets`` | Sets a custom snippet in the ``server`` context. | ``string`` | No | +{{}} + +\* -- Required for TLS Passthrough load balancing. + +### Listener + +The listener field references a listener that NGINX will use to accept incoming traffic for the TransportServer. For TCP and UDP, the listener must be defined in the [GlobalConfiguration resource]({{< ref "/nic/configuration/global-configuration/globalconfiguration-resource.md" >}}). When referencing a listener, both the name and the protocol must match. For TLS Passthrough, use the built-in listener with the name `tls-passthrough` and the protocol `TLS_PASSTHROUGH`. + +The combination of ``spec.listener.name`` and ``spec.host`` must be unique among all TransportServers. If two TransportServers specify the same combination of ``spec.listener.name`` and ``spec.host``, one of them will be rejected to prevent conflicts. In the case where no host is specified, it is considered an empty string. + +An example: + +```yaml +listener: + name: dns-udp + protocol: UDP +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the listener. | ``string`` | Yes | +|``protocol`` | The protocol of the listener. | ``string`` | Yes | +{{}} + +### TLS + +The tls field defines TLS configuration for a TransportServer. When using TLS termination (not TLS Passthrough), you can specify the host field to enable SNI-based routing, allowing multiple applications to share the same listener port and be distinguished by the TLS SNI hostname. Each application can use its own TLS certificate and key specified via the secret field. + +```yaml +secret: cafe-secret +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``secret`` | The name of a secret with a TLS certificate and key. The secret must belong to the same namespace as the TransportServer. The secret must be of the type ``kubernetes.io/tls`` and contain keys named ``tls.crt`` and ``tls.key`` that contain the certificate and private key as described [here](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls). | ``string`` | No | +{{}} + +### Upstream + +The upstream defines a destination for the TransportServer. For example: + +```yaml +name: secure-app +service: secure-app +port: 8443 +maxFails: 3 +maxConns: 100 +failTimeout: 30s +loadBalancingMethod: least_conn +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the upstream. Must be a valid DNS label as defined in RFC 1035. For example, ``hello`` and ``upstream-123`` are valid. The name must be unique among all upstreams of the resource. | ``string`` | Yes | +|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and close client connections/ignore datagrams. | ``string`` | Yes | +|``port`` | The port of the service. If the service doesn't define that port, NGINX will assume the service has zero endpoints and close client connections/ignore datagrams. The port must fall into the range ``1..65535``. | ``int`` | Yes | +|``maxFails`` | Sets the [number](https://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#max_fails) of unsuccessful attempts to communicate with the server that should happen in the duration set by the failTimeout parameter to consider the server unavailable. The default ``1``. | ``int`` | No | +|``maxConns`` | Sets the [number](https://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#max_conns) of maximum connections to the proxied server. Default value is zero, meaning there is no limit. The default is ``0``. | ``int`` | No | +|``failTimeout`` | Sets the [time](https://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#fail_timeout) during which the specified number of unsuccessful attempts to communicate with the server should happen to consider the server unavailable and the period of time the server will be considered unavailable. The default is ``10s``. | ``string`` | No | +|``healthCheck`` | The health check configuration for the Upstream. See the [health_check](https://nginx.org/en/docs/stream/ngx_stream_upstream_hc_module.html#health_check) directive. Note: this feature is supported only in NGINX Plus. | [healthcheck](#upstreamhealthcheck) | No | +|``loadBalancingMethod`` | The method used to load balance the upstream servers. By default, connections are distributed between the servers using a weighted round-robin balancing method. See the [upstream](http://nginx.org/en/docs/stream/ngx_stream_upstream_module.html#upstream) section for available methods and their details. | ``string`` | No | +|``backup`` | The name of the backup service of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). This will be used when the primary servers are unavailable. Note: The parameter cannot be used along with the ``random`` , ``hash`` or ``ip_hash`` load balancing methods. | ``string`` | No | +|``backupPort`` | The port of the backup service. The backup port is required if the backup service name is provided. The port must fall into the range ``1..65535``. | ``uint16`` | No | +{{}} + +### Upstream.Healthcheck + +The Healthcheck defines an [active health check](https://nginx.org/en/docs/stream/ngx_stream_upstream_hc_module.html?#health_check). In the example below we enable a health check for an upstream and configure all the available parameters: + +```yaml +name: secure-app +service: secure-app +port: 8443 +healthCheck: + enable: true + interval: 20s + timeout: 30s + jitter: 3s + fails: 5 + passes: 5 + port: 8080 +``` + +{{< note >}} This feature is only supported with NGINX Plus. {{< /note >}} + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables a health check for an upstream server. The default is ``false``. | ``boolean`` | No | +|``interval`` | The interval between two consecutive health checks. The default is ``5s``. | ``string`` | No | +|``timeout`` | This overrides the timeout set by [proxy_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout) which is set in `SessionParameters` for health checks. The default value is ``5s``. | ``string`` | No | +|``jitter`` | The time within which each health check will be randomly delayed. By default, there is no delay. | ``string`` | No | +|``fails`` | The number of consecutive failed health checks of a particular upstream server after which this server will be considered unhealthy. The default is ``1``. | ``integer`` | No | +|``passes`` | The number of consecutive passed health checks of a particular upstream server after which the server will be considered healthy. The default is ``1``. | ``integer`` | No | +|``port`` | The port used for health check requests. By default, the [server port is used](https://nginx.org/en/docs/stream/ngx_stream_upstream_hc_module.html#health_check_port). Note: in contrast with the port of the upstream, this port is not a service port, but a port of a pod. | ``integer`` | No | +|``match`` | Controls the data to send and the response to expect for the healthcheck. | [match](#upstreamhealthcheckmatch) | No | +{{}} + +### Upstream.Healthcheck.Match + +The match controls the data to send and the response to expect for the healthcheck: + +```yaml +match: + send: 'GET / HTTP/1.0\r\nHost: localhost\r\n\r\n' + expect: "~200 OK" +``` + +Both `send` and `expect` fields can contain hexadecimal literals with the prefix `\x` followed by two hex digits, for example, `\x80`. + +See the [match](https://nginx.org/en/docs/stream/ngx_stream_upstream_hc_module.html#match) directive for details. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``send`` | A string to send to an upstream server. | ``string`` | No | +|``expect`` | A literal string or a regular expression that the data obtained from the server should match. The regular expression is specified with the preceding ``~*`` modifier (for case-insensitive matching), or the ``~`` modifier (for case-sensitive matching). NGINX Ingress Controller validates a regular expression using the RE2 syntax. | ``string`` | No | +{{}} + +### UpstreamParameters + +The upstream parameters define various parameters for the upstreams: + +```yaml +upstreamParameters: + udpRequests: 1 + udpResponses: 1 + connectTimeout: 60s + nextUpstream: true + nextUpstreamTimeout: 50s + nextUpstreamTries: 1 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``udpRequests`` | The number of datagrams, after receiving which, the next datagram from the same client starts a new session. See the [proxy_requests](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_requests) directive. The default is ``0``. | ``int`` | No | +|``udpResponses`` | The number of datagrams expected from the proxied server in response to a client datagram. See the [proxy_responses](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_responses) directive. By default, the number of datagrams is not limited. | ``int`` | No | +|``connectTimeout`` | The timeout for establishing a connection with a proxied server. See the [proxy_connect_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_connect_timeout) directive. The default is ``60s``. | ``string`` | No | +|``nextUpstream`` | If a connection to the proxied server cannot be established, determines whether a client connection will be passed to the next server. See the [proxy_next_upstream](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream) directive. The default is ``true``. | bool | No | +|``nextUpstreamTries`` | The number of tries for passing a connection to the next server. See the [proxy_next_upstream_tries](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_tries) directive. The default is ``0``. | ``int`` | No | +|``nextUpstreamTimeout`` | The time allowed to pass a connection to the next server. See the [proxy_next_upstream_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_timeout) directive. The default us ``0``. | ``string`` | No | +{{}} + +### SessionParameters + +The session parameters define various parameters for TCP connections and UDP sessions. + +```yaml +sessionParameters: + timeout: 50s +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``timeout`` | The timeout between two successive read or write operations on client or proxied server connections. See [proxy_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout) directive. The default is ``10m``. | ``string`` | No | +{{}} + +### Action + +The action defines an action to perform for a client connection/datagram. + +In the example below, client connections/datagrams are passed to an upstream `dns-app`: + +```yaml +action: + pass: dns-app +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``pass`` | Passes connections/datagrams to an upstream. The upstream with that name must be defined in the resource. | ``string`` | Yes | +{{}} + +## Using TransportServer + +You can use the usual `kubectl` commands to work with TransportServer resources, similar to Ingress resources. + +For example, the following command creates a TransportServer resource defined in `transport-server-passthrough.yaml` with the name `secure-app`: + +```shell +kubectl apply -f transport-server-passthrough.yaml +``` +```text +transportserver.k8s.nginx.org/secure-app created +``` + +You can get the resource by running: + +```shell +kubectl get transportserver secure-app +``` +```text +NAME AGE +secure-app 46sm +``` + +In the kubectl get and similar commands, you can also use the short name `ts` instead of `transportserver`. + +### Using Snippets + +Snippets allow you to insert raw NGINX config into different contexts of NGINX configuration. In the example below, we use snippets to configure [access control](http://nginx.org/en/docs/stream/ngx_stream_access_module.html) in a TransportServer: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: TransportServer +metadata: + name: cafe +spec: + host: cafe.example.com + serverSnippets: | + deny 192.168.1.1; + allow 192.168.1.0/24; + upstreams: + - name: tea + service: tea-svc + port: 80 +``` + +Snippets can also be specified for a stream. In the example below, we use snippets to [limit the number of connections](https://nginx.org/en/docs/stream/ngx_stream_limit_conn_module.html): + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: TransportServer +metadata: + name: cafe +spec: + host: cafe.example.com + streamSnippets: limit_conn_zone $binary_remote_addr zone=addr:10m; + serverSnippets: limit_conn addr 1; + upstreams: + - name: tea + service: tea-svc + port: 80 +``` + +{{< note >}} To configure snippets in the `stream` context, use `stream-snippets` ConfigMap key. {{< /note >}} + +For additional information, view the [Advanced configuration with Snippets]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md" >}}) topic. + +### Validation + +Two types of validation are available for the TransportServer resource: + +- *Structural validation* by the `kubectl` and Kubernetes API server. +- *Comprehensive validation* by NGINX Ingress Controller. + +#### Structural Validation + +The custom resource definition for the TransportServer includes structural OpenAPI schema which describes the type of every field of the resource. + +If you try to create (or update) a resource that violates the structural schema (for example, you use a string value for the port field of an upstream), `kubectl` and Kubernetes API server will reject such a resource: + +- Example of `kubectl` validation: + + ```shell + kubectl apply -f transport-server-passthrough.yaml + ``` + ```text + error: error validating "transport-server-passthrough.yaml": error validating data: ValidationError(TransportServer.spec.upstreams[0].port): invalid type for org.nginx.k8s.v1.TransportServer.spec.upstreams.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + ``` + +- Example of Kubernetes API server validation: + + ```shell + kubectl apply -f transport-server-passthrough.yaml --validate=false + ``` + ```text + The TransportServer "secure-app" is invalid: []: Invalid value: map[string]interface {}{ ... }: validation failure list: + spec.upstreams.port in body must be of type integer: "string" + ``` + +If a resource is not rejected (it doesn't violate the structural schema), NGINX Ingress Controller will validate it further. + +#### Comprehensive Validation + +NGINX Ingress Controller validates the fields of a TransportServer resource. If a resource is invalid, NGINX Ingress Controller will reject it: the resource will continue to exist in the cluster, but NGINX Ingress Controller will ignore it. + +You can check if NGINX Ingress Controller successfully applied the configuration for a TransportServer. For our example `secure-app` TransportServer, we can run: + +```shell +kubectl describe ts secure-app +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 3s nginx-ingress-controller Configuration for default/secure-app was added or updated +``` + +Note how the events section includes a Normal event with the AddedOrUpdated reason that informs us that the configuration was successfully applied. + +If you create an invalid resource, NGINX Ingress Controller will reject it and emit a Rejected event. For example, if you create a TransportServer `secure-app` with a pass action that references a non-existing upstream, you will get : + +```shell +kubectl describe ts secure-app +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 2s nginx-ingress-controller TransportServer default/secure-app is invalid and was rejected: spec.action.pass: Not found: "some-app" +``` + +Note how the events section includes a Warning event with the Rejected reason. + +**Note**: If you make an existing resource invalid, NGINX Ingress Controller will reject it and remove the corresponding configuration from NGINX. + +## Customization via ConfigMap + +The [ConfigMap]({{< ref "/nic/configuration/global-configuration/configmap-resource.md" >}}) keys (except for `stream-snippets`, `stream-log-format`, `resolver-addresses`, `resolver-ipv6`, `resolver-valid` and `resolver-timeout`) do not affect TransportServer resources. diff --git a/content/nic/configuration/virtualserver-and-virtualserverroute-resources.md b/content/nic/configuration/virtualserver-and-virtualserverroute-resources.md new file mode 100644 index 000000000..1acd05b02 --- /dev/null +++ b/content/nic/configuration/virtualserver-and-virtualserverroute-resources.md @@ -0,0 +1,1101 @@ +--- +title: VirtualServer and VirtualServerRoute resources +toc: true +weight: 700 +docs: DOCS-599 +--- + +This document is reference material for the VirtualServer and VirtualServerRoute resources used by F5 NGINX Ingress Controller. + +VirtualServer and VirtualServerRoute resources are load balancing configurations recommended as an alternative to the Ingress resource. + +They enable use cases not supported with the Ingress resource, such as traffic splitting and advanced content-based routing. The resources are implemented as [Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + +The GitHub repository has [examples of the resources](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources) for specific use cases. + +--- + +## VirtualServer specification + +The VirtualServer resource defines load balancing configuration for a domain name, such as `example.com`. Below is an example of such configuration: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: cafe +spec: + host: cafe.example.com + listener: + http: http-8083 + https: https-8443 + tls: + secret: cafe-secret + gunzip: on + upstreams: + - name: tea + service: tea-svc + port: 80 + - name: coffee + service: coffee-svc + port: 80 + routes: + - path: /tea + action: + pass: tea + - path: /coffee + action: + pass: coffee + - path: ~ ^/decaf/.*\\.jpg$ + action: + pass: coffee + - path: = /green/tea + action: + pass: tea +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``host`` | The host (domain name) of the server. Must be a valid subdomain as defined in RFC 1123, such as ``my-app`` or ``hello.example.com``. When using a wildcard domain like ``*.example.com`` the domain must be contained in double quotes. The ``host`` value needs to be unique among all Ingress and VirtualServer resources. See also [Handling Host and Listener Collisions](/nginx-ingress-controller/configuration/host-and-listener-collisions). | ``string`` | Yes | +|``listener`` | Sets a custom HTTP and/or HTTPS listener. Valid fields are `listener.http` and `listener.https`. Each field must reference the name of a valid listener defined in a GlobalConfiguration resource | [listener](#virtualserverlistener) | No | +|``tls`` | The TLS termination configuration. | [tls](#virtualservertls) | No | +|``gunzip`` | Enables or disables [decompression](https://docs.nginx.com/nginx/admin-guide/web-server/compression/) of gzipped responses for clients. Allowed values “on”/“off”, “true”/“false” or “yes”/“no”. If the ``gunzip`` value is not set, it defaults to ``off``. | ``boolean`` | No | +|``externalDNS`` | The externalDNS configuration for a VirtualServer. | [externalDNS](#virtualserverexternaldns) | No | +|``dos`` | A reference to a DosProtectedResource, setting this enables DOS protection of the VirtualServer. | ``string`` | No | +|``policies`` | A list of policies. | [[]policy](#virtualserverpolicy) | No | +|``upstreams`` | A list of upstreams. | [[]upstream](#upstream) | No | +|``routes`` | A list of routes. | [[]route](#virtualserverroute) | No | +|``ingressClassName`` | Specifies which Ingress Controller must handle the VirtualServer resource. | ``string`` | No | +|``internalRoute`` | Specifies if the VirtualServer resource is an internal route or not. | ``boolean`` | No | +|``http-snippets`` | Sets a custom snippet in the http context. | ``string`` | No | +|``server-snippets`` | Sets a custom snippet in server context. Overrides the ``server-snippets`` ConfigMap key. | ``string`` | No | +{{}} + +### VirtualServer.TLS + +The tls field defines TLS configuration for a VirtualServer. For example: + +```yaml +secret: cafe-secret +redirect: + enable: true +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``secret`` | The name of a secret with a TLS certificate and key. The secret must belong to the same namespace as the VirtualServer. The secret must be of the type ``kubernetes.io/tls`` and contain keys named ``tls.crt`` and ``tls.key`` that contain the certificate and private key as described [here](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls). If the secret doesn't exist or is invalid, NGINX will break any attempt to establish a TLS connection to the host of the VirtualServer. If the secret is not specified but [wildcard TLS secret](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments#cmdoption-wildcard-tls-secret) is configured, NGINX will use the wildcard secret for TLS termination. | ``string`` | No | +|``redirect`` | The redirect configuration of the TLS for a VirtualServer. | [tls.redirect](#virtualservertlsredirect) | No | ### VirtualServer.TLS.Redirect | +|``cert-manager`` | The cert-manager configuration of the TLS for a VirtualServer. | [tls.cert-manager](#virtualservertlscertmanager) | No | ### VirtualServer.TLS.CertManager | +{{}} + +### VirtualServer.TLS.Redirect + +The redirect field configures a TLS redirect for a VirtualServer: + +```yaml +enable: true +code: 301 +basedOn: scheme +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables a TLS redirect for a VirtualServer. The default is ``False``. | ``boolean`` | No | +|``code`` | The status code of a redirect. The allowed values are: ``301`` , ``302`` , ``307`` , ``308``. The default is ``301``. | ``int`` | No | +|``basedOn`` | The attribute of a request that NGINX will evaluate to send a redirect. The allowed values are ``scheme`` (the scheme of the request) or ``x-forwarded-proto`` (the ``X-Forwarded-Proto`` header of the request). The default is ``scheme``. | ``string`` | No | ### VirtualServer.Policy | +{{}} + +### VirtualServer.TLS.CertManager + +The cert-manager field configures x509 automated Certificate management for VirtualServer resources using cert-manager (cert-manager.io). Please see the [cert-manager configuration documentation](https://cert-manager.io/docs/configuration/) for more information on deploying and configuring Issuers. Example: + +```yaml +cert-manager: + cluster-issuer: "my-issuer-name" +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``issuer`` | the name of an Issuer. An Issuer is a cert-manager resource which describes the certificate authority capable of signing certificates. The Issuer must be in the same namespace as the VirtualServer resource. Please note that one of `issuer` and `cluster-issuer` are required, but they are mutually exclusive - one and only one must be defined. | ``string`` | No | +|``cluster-issuer`` | the name of a ClusterIssuer. A ClusterIssuer is a cert-manager resource which describes the certificate authority capable of signing certificates. It does not matter which namespace your VirtualServer resides, as ClusterIssuers are non-namespaced resources. Please note that one of `issuer` and `cluster-issuer` are required, but they are mutually exclusive - one and only one must be defined. | ``string`` | No | +|``issuer-kind`` | The kind of the external issuer resource, for example AWSPCAIssuer. This is only necessary for out-of-tree issuers. This cannot be defined if `cluster-issuer` is also defined. | ``string`` | No | +|``issuer-group`` | The API group of the external issuer controller, for example awspca.cert-manager.io. This is only necessary for out-of-tree issuers. This cannot be defined if `cluster-issuer` is also defined. | ``string`` | No | +|``common-name`` | This field allows you to configure spec.commonName for the Certificate to be generated. This configuration adds a CN to the x509 certificate. | ``string`` | No | +|``duration`` | This field allows you to configure spec.duration field for the Certificate to be generated. Must be specified using a [Go time.Duration](https://pkg.go.dev/time#ParseDuration) string format, which does not allow the d (days) suffix. You must specify these values using s, m, and h suffixes instead. | ``string`` | No | +|``renew-before`` | this annotation allows you to configure spec.renewBefore field for the Certificate to be generated. Must be specified using a [Go time.Duration](https://pkg.go.dev/time#ParseDuration) string format, which does not allow the d (days) suffix. You must specify these values using s, m, and h suffixes instead. | ``string`` | No | +|``usages`` | This field allows you to configure spec.usages field for the Certificate to be generated. Pass a string with comma-separated values i.e. ``key agreement,digital signature, server auth``. An exhaustive list of supported key usages can be found in the [the cert-manager api documentation](https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.KeyUsage). | ``string`` | No | +|``issue-temp-cert`` | When ``true``, ask cert-manager for a [temporary self-signed certificate](https://cert-manager.io/docs/usage/certificate/#temporary-certificates-while-issuing) pending the issuance of the Certificate. This allows HTTPS-only servers to use ACME HTTP01 challenges when the TLS secret does not exist yet. | ``boolean`` | No | +{{}} + +### VirtualServer.Listener +The listener field defines a custom HTTP and/or HTTPS listener. +The respective listeners used must reference the name of a listener defined using a [GlobalConfiguration](/nginx-ingress-controller/configuration/global-configuration/globalconfiguration-resource/) resource. +For example: +```yaml +http: http-8083 +https: https-8443 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``http`` | The name of am HTTP listener defined in a [GlobalConfiguration](/nginx-ingress-controller/configuration/global-configuration/globalconfiguration-resource/) resource. | ``string`` | No | +|``https`` | The name of an HTTPS listener defined in a [GlobalConfiguration](/nginx-ingress-controller/configuration/global-configuration/globalconfiguration-resource/) resource. | ``string`` | No | +{{}} + +### VirtualServer.ExternalDNS + +The externalDNS field configures controlling DNS records dynamically for VirtualServer resources using [ExternalDNS](https://github.com/kubernetes-sigs/external-dns). Please see the [ExternalDNS configuration documentation](https://kubernetes-sigs.github.io/external-dns/) for more information on deploying and configuring ExternalDNS and Providers. Example: + +```yaml +enable: true +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables ExternalDNS integration for a VirtualServer resource. The default is ``false``. | ``string`` | No | +|``labels`` | Configure labels to be applied to the Endpoint resources that will be consumed by ExternalDNS. | ``map[string]string`` | No | +|``providerSpecific`` | Configure provider specific properties which holds the name and value of a configuration which is specific to individual DNS providers. | [[]ProviderSpecific](#virtualserverexternaldnsproviderspecific) | No | +|``recordTTL`` | TTL for the DNS record. This defaults to 0 if not defined. See [the ExternalDNS TTL documentation for provider-specific defaults](https://kubernetes-sigs.github.io/external-dns/v0.14.2/ttl/#providers) | ``int64`` | No | +|``recordType`` | The record Type that should be created, e.g. "A", "AAAA", "CNAME". This is automatically computed based on the external endpoints if not defined. | ``string`` | No | +{{}} + +### VirtualServer.ExternalDNS.ProviderSpecific + +The providerSpecific field of the externalDNS block allows the specification of provider specific properties which is a list of key value pairs of configurations which are specific to individual DNS providers. Example: + +```yaml +- name: my-name + value: my-value +- name: my-name2 + value: my-value2 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the key value pair. | ``string`` | Yes | +|``value`` | The value of the key value pair. | ``string`` | Yes | +{{}} + +### VirtualServer.Policy + +The policy field references a [Policy resource](/nginx-ingress-controller/configuration/policy-resource/) by its name and optional namespace. For example: + +```yaml +name: access-control +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of a policy. If the policy doesn't exist or invalid, NGINX will respond with an error response with the `500` status code. | ``string`` | Yes | +|``namespace`` | The namespace of a policy. If not specified, the namespace of the VirtualServer resource is used. | ``string`` | No | +{{}} + +### VirtualServer.Route + +The route defines rules for matching client requests to actions like passing a request to an upstream. For example: + +```yaml + path: /tea + action: + pass: tea +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``path`` | The path of the route. NGINX will match it against the URI of a request. Possible values are: a prefix ( ``/`` , ``/path`` ), an exact match ( ``=/exact/match`` ), a case insensitive regular expression ( ``~*^/Bar.*\.jpg`` ) or a case sensitive regular expression ( ``~^/foo.*\.jpg`` ). In the case of a prefix (must start with ``/`` ) or an exact match (must start with ``=`` ), the path must not include any whitespace characters, ``{`` , ``}`` or ``;``. In the case of the regex matches, all double quotes ``"`` must be escaped and the match can't end in an unescaped backslash ``\``. The path must be unique among the paths of all routes of the VirtualServer. Check the [location](https://nginx.org/en/docs/http/ngx_http_core_module.html#location) directive for more information. | ``string`` | Yes | +|``policies`` | A list of policies. The policies override the policies of the same type defined in the ``spec`` of the VirtualServer. See [Applying Policies](/nginx-ingress-controller/configuration/policy-resource/#applying-policies) for more details. | [[]policy](#virtualserverpolicy) | No | +|``action`` | The default action to perform for a request. | [action](#action) | No | +|``dos`` | A reference to a DosProtectedResource, setting this enables DOS protection of the VirtualServer route. | ``string`` | No | +|``splits`` | The default splits configuration for traffic splitting. Must include at least 2 splits. | [[]split](#split) | No | +|``matches`` | The matching rules for advanced content-based routing. Requires the default ``action`` or ``splits``. Unmatched requests will be handled by the default ``action`` or ``splits``. | [matches](#match) | No | +|``route`` | The name of a VirtualServerRoute resource that defines this route. If the VirtualServerRoute belongs to a different namespace than the VirtualServer, you need to include the namespace. For example, ``tea-namespace/tea``. | ``string`` | No | +|``errorPages`` | The custom responses for error codes. NGINX will use those responses instead of returning the error responses from the upstream servers or the default responses generated by NGINX. A custom response can be a redirect or a canned response. For example, a redirect to another URL if an upstream server responded with a 404 status code. | [[]errorPage](#errorpage) | No | +|``location-snippets`` | Sets a custom snippet in the location context. Overrides the ``location-snippets`` ConfigMap key. | ``string`` | No | +{{}} + +\* -- a route must include exactly one of the following: `action`, `splits`, or `route`. + +## VirtualServerRoute specification + +The VirtualServerRoute resource defines a route for a VirtualServer. It can consist of one or multiple subroutes. The VirtualServerRoute is an alternative to [Mergeable Ingress types](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration). + +In the example below, the VirtualServer `cafe` from the namespace `cafe-ns` defines a route with the path `/coffee`, which is further defined in the VirtualServerRoute `coffee` from the namespace `coffee-ns`. + +VirtualServer: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: cafe + namespace: cafe-ns +spec: + host: cafe.example.com + upstreams: + - name: tea + service: tea-svc + port: 80 + routes: + - path: /tea + action: + pass: tea + - path: /coffee + route: coffee-ns/coffee +``` + +VirtualServerRoute: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServerRoute +metadata: + name: coffee + namespace: coffee-ns +spec: + host: cafe.example.com + upstreams: + - name: latte + service: latte-svc + port: 80 + - name: espresso + service: espresso-svc + port: 80 + subroutes: + - path: /coffee/latte + action: + pass: latte + - path: /coffee/espresso + action: + pass: espresso +``` + +Note that each subroute must have a `path` that starts with the same prefix (here `/coffee`), which is defined in the route of the VirtualServer. Additionally, the `host` in the VirtualServerRoute must be the same as the `host` of the VirtualServer. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``host`` | The host (domain name) of the server. Must be a valid subdomain as defined in RFC 1123, such as ``my-app`` or ``hello.example.com``. When using a wildcard domain like ``*.example.com`` the domain must be contained in double quotes. Must be the same as the ``host`` of the VirtualServer that references this resource. | ``string`` | Yes | +|``upstreams`` | A list of upstreams. | [[]upstream](#upstream) | No | +|``subroutes`` | A list of subroutes. | [[]subroute](#virtualserverroutesubroute) | No | +|``ingressClassName`` | Specifies which Ingress Controller must handle the VirtualServerRoute resource. Must be the same as the ``ingressClassName`` of the VirtualServer that references this resource. | ``string``_ | No | +{{}} + +### VirtualServerRoute.Subroute + +The subroute defines rules for matching client requests to actions like passing a request to an upstream. For example: + +```yaml +path: /coffee +action: + pass: coffee +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``path`` | The path of the subroute. NGINX will match it against the URI of a request. Possible values are: a prefix ( ``/`` , ``/path`` ), an exact match ( ``=/exact/match`` ), a case insensitive regular expression ( ``~*^/Bar.*\.jpg`` ) or a case sensitive regular expression ( ``~^/foo.*\.jpg`` ). In the case of a prefix, the path must start with the same path as the path of the route of the VirtualServer that references this resource. In the case of an exact or regex match, the path must be the same as the path of the route of the VirtualServer that references this resource. A matching path of the route of the VirtualServer but in different type is not accepted, e.g. a regex path (`~/match`) cannot be used with a prefix path in VirtualServer (`/match`) In the case of a prefix or an exact match, the path must not include any whitespace characters, ``{`` , ``}`` or ``;``. In the case of the regex matches, all double quotes ``"`` must be escaped and the match can't end in an unescaped backslash ``\``. The path must be unique among the paths of all subroutes of the VirtualServerRoute. | ``string`` | Yes | +|``policies`` | A list of policies. The policies override *all* policies defined in the route of the VirtualServer that references this resource. The policies also override the policies of the same type defined in the ``spec`` of the VirtualServer. See [Applying Policies](/nginx-ingress-controller/configuration/policy-resource/#applying-policies) for more details. | [[]policy](#virtualserverpolicy) | No | +|``action`` | The default action to perform for a request. | [action](#action) | No | +|``dos`` | A reference to a DosProtectedResource, setting this enables DOS protection of the VirtualServerRoute subroute. | ``string`` | No | +|``splits`` | The default splits configuration for traffic splitting. Must include at least 2 splits. | [[]split](#split) | No | +|``matches`` | The matching rules for advanced content-based routing. Requires the default ``action`` or ``splits``. Unmatched requests will be handled by the default ``action`` or ``splits``. | [matches](#match) | No | +|``errorPages`` | The custom responses for error codes. NGINX will use those responses instead of returning the error responses from the upstream servers or the default responses generated by NGINX. A custom response can be a redirect or a canned response. For example, a redirect to another URL if an upstream server responded with a 404 status code. | [[]errorPage](#errorpage) | No | +|``location-snippets`` | Sets a custom snippet in the location context. Overrides the ``location-snippets`` of the VirtualServer (if set) or the ``location-snippets`` ConfigMap key. | ``string`` | No | +{{}} + +\* -- a subroute must include exactly one of the following: `action` or `splits`. + +## Common VirtualServer and VirtualServerRoute specifications + +### Upstream + +The upstream defines a destination for the routing configuration. For example: + +```yaml +name: tea +service: tea-svc +subselector: + version: canary +port: 80 +lb-method: round_robin +fail-timeout: 10s +max-fails: 1 +max-conns: 32 +keepalive: 32 +connect-timeout: 30s +read-timeout: 30s +send-timeout: 30s +next-upstream: "error timeout non_idempotent" +next-upstream-timeout: 5s +next-upstream-tries: 10 +client-max-body-size: 2m +tls: + enable: true +``` + +**Note**: The WebSocket protocol is supported without any additional configuration. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the upstream. Must be a valid DNS label as defined in RFC 1035. For example, ``hello`` and ``upstream-123`` are valid. The name must be unique among all upstreams of the resource. | ``string`` | Yes | +|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. For NGINX Plus only, services of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) are also supported (check the [prerequisites](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/ingress-resources/externalname-services#prerequisites) ). | ``string`` | Yes | +|``subselector`` | Selects the pods within the service using label keys and values. By default, all pods of the service are selected. Note: the specified labels are expected to be present in the pods when they are created. If the pod labels are updated, NGINX Ingress Controller will not see that change until the number of the pods is changed. | ``map[string]string`` | No | +|``use-cluster-ip`` | Enables using the Cluster IP and port of the service instead of the default behavior of using the IP and port of the pods. When this field is enabled, the fields that configure NGINX behavior related to multiple upstream servers (like ``lb-method`` and ``next-upstream``) will have no effect, as NGINX Ingress Controller will configure NGINX with only one upstream server that will match the service Cluster IP. | ``boolean`` | No | +|``port`` | The port of the service. If the service doesn't define that port, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. The port must fall into the range ``1..65535``. | ``uint16`` | Yes | +|``lb-method`` | The load [balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify ``round_robin``. The default is specified in the ``lb-method`` ConfigMap key. | ``string`` | No | +|``fail-timeout`` | The time during which the specified number of unsuccessful attempts to communicate with an upstream server should happen to consider the server unavailable. See the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the server directive. The default is set in the ``fail-timeout`` ConfigMap key. | ``string`` | No | +|``max-fails`` | The number of unsuccessful attempts to communicate with an upstream server that should happen in the duration set by the ``fail-timeout`` to consider the server unavailable. See the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the server directive. The default is set in the ``max-fails`` ConfigMap key. | ``int`` | No | +|``max-conns`` | The maximum number of simultaneous active connections to an upstream server. See the [max_conns](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_conns) parameter of the server directive. By default there is no limit. Note: if keepalive connections are enabled, the total number of active and idle keepalive connections to an upstream server may exceed the ``max_conns`` value. | ``int`` | No | +|``keepalive`` | Configures the cache for connections to upstream servers. The value ``0`` disables the cache. See the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. The default is set in the ``keepalive`` ConfigMap key. | ``int`` | No | +|``connect-timeout`` | The timeout for establishing a connection with an upstream server. See the [proxy_connect_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) directive. The default is specified in the ``proxy-connect-timeout`` ConfigMap key. | ``string`` | No | +|``read-timeout`` | The timeout for reading a response from an upstream server. See the [proxy_read_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) directive. The default is specified in the ``proxy-read-timeout`` ConfigMap key. | ``string`` | No | +|``send-timeout`` | The timeout for transmitting a request to an upstream server. See the [proxy_send_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout) directive. The default is specified in the ``proxy-send-timeout`` ConfigMap key. | ``string`` | No | +|``next-upstream`` | Specifies in which cases a request should be passed to the next upstream server. See the [proxy_next_upstream](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream) directive. The default is ``error timeout``. | ``string`` | No | +|``next-upstream-timeout`` | The time during which a request can be passed to the next upstream server. See the [proxy_next_upstream_timeout](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream_timeout) directive. The ``0`` value turns off the time limit. The default is ``0``. | ``string`` | No | +|``next-upstream-tries`` | The number of possible tries for passing a request to the next upstream server. See the [proxy_next_upstream_tries](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream_tries) directive. The ``0`` value turns off this limit. The default is ``0``. | ``int`` | No | +|``client-max-body-size`` | Sets the maximum allowed size of the client request body. See the [client_max_body_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) directive. The default is set in the ``client-max-body-size`` ConfigMap key. | ``string`` | No | +|``tls`` | The TLS configuration for the Upstream. | [tls](#upstreamtls) | No | +|``healthCheck`` | The health check configuration for the Upstream. See the [health_check](https://nginx.org/en/docs/http/ngx_http_upstream_hc_module.html#health_check) directive. Note: this feature is supported only in NGINX Plus. | [healthcheck](#upstreamhealthcheck) | No | +|``slow-start`` | The slow start allows an upstream server to gradually recover its weight from 0 to its nominal value after it has been recovered or became available or when the server becomes available after a period of time it was considered unavailable. By default, the slow start is disabled. See the [slow_start](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#slow_start) parameter of the server directive. Note: The parameter cannot be used along with the ``random`` , ``hash`` or ``ip_hash`` load balancing methods and will be ignored. | ``string`` | No | +|``queue`` | Configures a queue for an upstream. A client request will be placed into the queue if an upstream server cannot be selected immediately while processing the request. By default, no queue is configured. Note: this feature is supported only in NGINX Plus. | [queue](#upstreamqueue) | No | +|``buffering`` | Enables buffering of responses from the upstream server. See the [proxy_buffering](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) directive. The default is set in the ``proxy-buffering`` ConfigMap key. | ``boolean`` | No | +|``buffers`` | Configures the buffers used for reading a response from the upstream server for a single connection. | [buffers](#upstreambuffers) | No | +|``buffer-size`` | Sets the size of the buffer used for reading the first part of a response received from the upstream server. See the [proxy_buffer_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) directive. The default is set in the ``proxy-buffer-size`` ConfigMap key. | ``string`` | No | +|``ntlm`` | Allows proxying requests with NTLM Authentication. See the [ntlm](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#ntlm) directive. In order for NTLM authentication to work, it is necessary to enable keepalive connections to upstream servers using the ``keepalive`` field. Note: this feature is supported only in NGINX Plus.| ``boolean`` | No | +|``type`` |The type of the upstream. Supported values are ``http`` and ``grpc``. The default is ``http``. For gRPC, it is necessary to enable HTTP/2 in the [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#listeners) and configure TLS termination in the VirtualServer. | ``string`` | No | +|``backup`` | The name of the backup service of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). This will be used when the primary servers are unavailable. Note: The parameter cannot be used along with the ``random`` , ``hash`` or ``ip_hash`` load balancing methods. | ``string`` | No | +|``backupPort`` | The port of the backup service. The backup port is required if the backup service name is provided. The port must fall into the range ``1..65535``. | ``uint16`` | No | +{{}} + +### Upstream.Buffers + +The buffers field configures the buffers used for reading a response from the upstream server for a single connection: + +```yaml +number: 4 +size: 8K +``` + +See the [proxy_buffers](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directive for additional information. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``number`` | Configures the number of buffers. The default is set in the ``proxy-buffers`` ConfigMap key. | ``int`` | Yes | +|``size`` | Configures the size of a buffer. The default is set in the ``proxy-buffers`` ConfigMap key. | ``string`` | Yes | +{{}} + +### Upstream.TLS + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables HTTPS for requests to upstream servers. The default is ``False`` , meaning that HTTP will be used. Note: by default, NGINX will not verify the upstream server certificate. To enable the verification, configure an [EgressMTLS Policy](/nginx-ingress-controller/configuration/policy-resource/#egressmtls). | ``boolean`` | No | +{{}} + +### Upstream.Queue + +The queue field configures a queue. A client request will be placed into the queue if an upstream server cannot be selected immediately while processing the request: + +```yaml +size: 10 +timeout: 60s +``` + +See [`queue`](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#queue) directive for additional information. + +Note: This feature is supported only in NGINX Plus. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``size`` | The size of the queue. | ``int`` | Yes | +|``timeout`` | The timeout of the queue. A request cannot be queued for a period longer than the timeout. The default is ``60s``. | ``string`` | No | +{{}} + +### Upstream.Healthcheck + +The Healthcheck defines an [active health check](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/). In the example below we enable a health check for an upstream and configure all the available parameters, including the `slow-start` parameter combined with [`mandatory` and `persistent`](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#mandatory-health-checks): + +```yaml +name: tea +service: tea-svc +port: 80 +slow-start: 30s +healthCheck: + enable: true + path: /healthz + interval: 20s + jitter: 3s + fails: 5 + passes: 5 + port: 8080 + tls: + enable: true + connect-timeout: 10s + read-timeout: 10s + send-timeout: 10s + headers: + - name: Host + value: my.service + statusMatch: "! 500" + mandatory: true + persistent: true + keepalive-time: 60s +``` + +Note: This feature is supported only in NGINX Plus. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables a health check for an upstream server. The default is ``false``. | ``boolean`` | No | +|``path`` | The path used for health check requests. The default is ``/``. This not configurable for gRPC type upstreams. | ``string`` | No | +|``interval`` | The interval between two consecutive health checks. The default is ``5s``. | ``string`` | No | +|``jitter`` | The time within which each health check will be randomly delayed. By default, there is no delay. | ``string`` | No | +|``fails`` | The number of consecutive failed health checks of a particular upstream server after which this server will be considered unhealthy. The default is ``1``. | ``integer`` | No | +|``passes`` | The number of consecutive passed health checks of a particular upstream server after which the server will be considered healthy. The default is ``1``. | ``integer`` | No | +|``port`` | The port used for health check requests. By default, the [server port is used](https://nginx.org/en/docs/http/ngx_http_upstream_hc_module.html#health_check_port). Note: in contrast with the port of the upstream, this port is not a service port, but a port of a pod. | ``integer`` | No | +|``tls`` | The TLS configuration used for health check requests. By default, the ``tls`` field of the upstream is used. | [upstream.tls](#upstreamtls) | No | +|``connect-timeout`` | The timeout for establishing a connection with an upstream server. By default, the ``connect-timeout`` of the upstream is used. | ``string`` | No | +|``read-timeout`` | The timeout for reading a response from an upstream server. By default, the ``read-timeout`` of the upstream is used. | ``string`` | No | +|``send-timeout`` | The timeout for transmitting a request to an upstream server. By default, the ``send-timeout`` of the upstream is used. | ``string`` | No | +|``headers`` | The request headers used for health check requests. NGINX Plus always sets the ``Host`` , ``User-Agent`` and ``Connection`` headers for health check requests. | [[]header](#header) | No | +|``statusMatch`` | The expected response status codes of a health check. By default, the response should have status code 2xx or 3xx. Examples: ``"200"`` , ``"! 500"`` , ``"301-303 307"``. See the documentation of the [match](https://nginx.org/en/docs/http/ngx_http_upstream_hc_module.html?#match) directive. This not supported for gRPC type upstreams. | ``string`` | No | +|``grpcStatus`` | The expected [gRPC status code](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md#status-codes-and-their-use-in-grpc) of the upstream server response to the [Check method](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). Configure this field only if your gRPC services do not implement the gRPC health checking protocol. For example, configure ``12`` if the upstream server responds with `12 (UNIMPLEMENTED)` status code. Only valid on gRPC type upstreams. | ``int`` | No | +|``grpcService`` | The gRPC service to be monitored on the upstream server. Only valid on gRPC type upstreams. | ``string`` | No | +|``mandatory`` | Require every newly added server to pass all configured health checks before NGINX Plus sends traffic to it. If this is not specified, or is set to false, the server will be initially considered healthy. When combined with [slow-start](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#slow_start), it gives a new server more time to connect to databases and “warm up” before being asked to handle their full share of traffic. | ``bool`` | No | +|``persistent`` | Set the initial “up” state for a server after reload if the server was considered healthy before reload. Enabling persistent requires that the mandatory parameter is also set to `true`. | ``bool`` | No | +|``keepalive-time`` | Enables [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) connections for health checks and specifies the time during which requests can be processed through one keepalive connection. The default is ``60s``. | ``string`` | No | +{{}} + +### Upstream.SessionCookie + +The SessionCookie field configures session persistence which allows requests from the same client to be passed to the same upstream server. The information about the designated upstream server is passed in a session cookie generated by NGINX Plus. + +In the example below, we configure session persistence with a session cookie for an upstream and configure all the available parameters: + +```yaml +name: tea +service: tea-svc +port: 80 +sessionCookie: + enable: true + name: srv_id + path: / + expires: 1h + domain: .example.com + httpOnly: false + secure: true + samesite: strict +``` + +See the [`sticky`](https://nginx.org/en/docs/http/ngx_http_upstream_module.html?#sticky) directive for additional information. The session cookie corresponds to the `sticky cookie` method. + +Note: This feature is supported only in NGINX Plus. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``enable`` | Enables session persistence with a session cookie for an upstream server. The default is ``false``. | ``boolean`` | No | +|``name`` | The name of the cookie. | ``string`` | Yes | +|``path`` | The path for which the cookie is set. | ``string`` | No | +|``expires`` | The time for which a browser should keep the cookie. Can be set to the special value ``max`` , which will cause the cookie to expire on ``31 Dec 2037 23:55:55 GMT``. | ``string`` | No | +|``domain`` | The domain for which the cookie is set. | ``string`` | No | +|``httpOnly`` | Adds the ``HttpOnly`` attribute to the cookie. | ``boolean`` | No | +|``secure`` | Adds the ``Secure`` attribute to the cookie. | ``boolean`` | No | +|``samesite`` | Adds the ``SameSite`` attribute to the cookie. The allowed values are: ``strict``, ``lax``, ``none`` | ``string`` | No | +{{}} + +### Header + +The header defines an HTTP Header: + +```yaml +name: Host +value: example.com +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. | ``string`` | No | +{{}} + +### Action + +The action defines an action to perform for a request. + +In the example below, client requests are passed to an upstream `coffee`: + +```yaml + path: /coffee + action: + pass: coffee +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``pass`` | Passes requests to an upstream. The upstream with that name must be defined in the resource. | ``string`` | No | +|``redirect`` | Redirects requests to a provided URL. | [action.redirect](#actionredirect) | No | +|``return`` | Returns a preconfigured response. | [action.return](#actionreturn) | No | +|``proxy`` | Passes requests to an upstream with the ability to modify the request/response (for example, rewrite the URI or modify the headers). | [action.proxy](#actionproxy) | No | +{{}} + +\* -- an action must include exactly one of the following: `pass`, `redirect`, `return` or `proxy`. + +### Action.Redirect + +The redirect action defines a redirect to return for a request. + +In the example below, client requests are passed to a url `http://www.nginx.com`: + +```yaml +redirect: + url: http://www.nginx.com + code: 301 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``url`` | The URL to redirect the request to. Supported NGINX variables: ``$scheme`` , ``$http_x_forwarded_proto`` , ``$request_uri`` , ``$host``. Variables must be enclosed in curly braces. For example: ``${host}${request_uri}``. | ``string`` | Yes | +|``code`` | The status code of a redirect. The allowed values are: ``301`` , ``302`` , ``307`` , ``308``. The default is ``301``. | ``int`` | No | +{{}} + +### Action.Return + +The return action defines a preconfigured response for a request. + +In the example below, NGINX will respond with the preconfigured response for every request: + +```yaml +return: + code: 200 + type: text/plain + body: "Hello World\n" + headers: + - name: x-coffee + value: espresso +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``code`` | The status code of the response. The allowed values are: ``2XX``, ``4XX`` or ``5XX``. The default is ``200``. | ``int`` | No | +|``type`` | The MIME type of the response. The default is ``text/plain``. | ``string`` | No | +|``body`` | The body of the response. Supports NGINX variables*. Variables must be enclosed in curly brackets. For example: ``Request is ${request_uri}\n``. | ``string`` | Yes | +|``headers`` | The custom headers of the response. | [[]Action.Return.Header](#actionreturnheader) | No | +{{}} + +\* -- Supported NGINX variables: `$request_uri`, `$request_method`, `$request_body`, `$scheme`, `$http_`, `$args`, `$arg_`, `$cookie_`, `$host`, `$request_time`, `$request_length`, `$nginx_version`, `$pid`, `$connection`, `$remote_addr`, `$remote_port`, `$time_iso8601`, `$time_local`, `$server_addr`, `$server_port`, `$server_name`, `$server_protocol`, `$connections_active`, `$connections_reading`, `$connections_writing` and `$connections_waiting`. + +### Action.Return.Header + +The header defines an HTTP Header for a canned response in an actionReturn: + +```yaml +name: x-coffee +value: espresso +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. | ``string`` | Yes | +{{}} + +### Action.Proxy + +The proxy action passes requests to an upstream with the ability to modify the request/response (for example, rewrite the URI or modify the headers). + +In the example below, the request URI is rewritten to `/`, and the request and the response headers are modified: + +```yaml +proxy: + upstream: coffee + requestHeaders: + pass: true + set: + - name: My-Header + value: Value + - name: Client-Cert + value: ${ssl_client_escaped_cert} + responseHeaders: + add: + - name: My-Header + value: Value + - name: IC-Nginx-Version + value: ${nginx_version} + always: true + hide: + - x-internal-version + ignore: + - Expires + - Set-Cookie + pass: + - Server + rewritePath: / +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``upstream`` | The name of the upstream which the requests will be proxied to. The upstream with that name must be defined in the resource. | ``string`` | Yes | +|``requestHeaders`` | The request headers modifications. | [action.Proxy.RequestHeaders](#actionproxyrequestheaders) | No | +|``responseHeaders`` | The response headers modifications. | [action.Proxy.ResponseHeaders](#actionproxyresponseheaders) | No | +|``rewritePath`` | The rewritten URI. If the route path is a regular expression -- starts with `~` -- the `rewritePath` can include capture groups with ``$1-9``. For example `$1` for the first group, and so on. For more information, check the [rewrite](https://github.com/nginx/kubernetes-ingress/tree/v{{< nic-version >}}/examples/custom-resources/rewrites) example. | ``string`` | No | +{{}} + +### Action.Proxy.RequestHeaders + +The RequestHeaders field modifies the headers of the request to the proxied upstream server. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``pass`` | Passes the original request headers to the proxied upstream server. See the [proxy_pass_request_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_request_headers) directive for more information. Default is true. | ``bool`` | No | +|``set`` | Allows redefining or appending fields to present request headers passed to the proxied upstream servers. See the [proxy_set_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header) directive for more information. | [[]header](#actionproxyrequestheaderssetheader) | No | +{{}} + +### Action.Proxy.RequestHeaders.Set.Header + +The header defines an HTTP Header: + +```yaml +name: My-Header +value: My-Value +``` + +It is possible to override the default value of the `Host` header, which NGINX Ingress Controller sets to [`$host`](https://nginx.org/en/docs/http/ngx_http_core_module.html#var_host): + +```yaml +name: Host +value: example.com +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. Supports NGINX variables*. Variables must be enclosed in curly brackets. For example: ``${scheme}``. | ``string`` | No | +{{}} + +\* -- Supported NGINX variables: `$request_uri`, `$request_method`, `$request_body`, `$scheme`, `$http_`, `$args`, `$arg_`, `$cookie_`, `$host`, `$request_time`, `$request_length`, `$nginx_version`, `$pid`, `$connection`, `$remote_addr`, `$remote_port`, `$time_iso8601`, `$time_local`, `$server_addr`, `$server_port`, `$server_name`, `$server_protocol`, `$connections_active`, `$connections_reading`, `$connections_writing`, `$connections_waiting`, `$ssl_cipher`, `$ssl_ciphers`, `$ssl_client_cert`, `$ssl_client_escaped_cert`, `$ssl_client_fingerprint`, `$ssl_client_i_dn`, `$ssl_client_i_dn_legacy`, `$ssl_client_raw_cert`, `$ssl_client_s_dn`, `$ssl_client_s_dn_legacy`, `$ssl_client_serial`, `$ssl_client_v_end`, `$ssl_client_v_remain`, `$ssl_client_v_start`, `$ssl_client_verify`, `$ssl_curves`, `$ssl_early_data`, `$ssl_protocol`, `$ssl_server_name`, `$ssl_session_id`, `$ssl_session_reused`, `$jwt_claim_` (NGINX Plus only) and `$jwt_header_` (NGINX Plus only). + +### Action.Proxy.ResponseHeaders + +The ResponseHeaders field modifies the headers of the response to the client. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``hide`` | The headers that will not be passed* in the response to the client from a proxied upstream server. See the [proxy_hide_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directive for more information. | ``[]string`` | No | +|``pass`` | Allows passing the hidden header fields* to the client from a proxied upstream server. See the [proxy_pass_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directive for more information. | ``[]string`` | No | +|``ignore`` | Disables processing of certain headers** to the client from a proxied upstream server. See the [proxy_ignore_headers](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ignore_headers) directive for more information. | ``[]string`` | No | +|``add`` | Adds headers to the response to the client. | [[]addHeader](#addheader) | No | +{{}} + +\* -- Default hidden headers are: `Date`, `Server`, `X-Pad` and `X-Accel-...`. + +\** -- The following fields can be ignored: `X-Accel-Redirect`, `X-Accel-Expires`, `X-Accel-Limit-Rate`, `X-Accel-Buffering`, `X-Accel-Charset`, `Expires`, `Cache-Control`, `Set-Cookie` and `Vary`. + +### AddHeader + +The addHeader defines an HTTP Header with an optional `always` field: + +```yaml +name: My-Header +value: My-Value +always: true +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. Supports NGINX variables*. Variables must be enclosed in curly brackets. For example: ``${scheme}``. | ``string`` | No | +|``always`` | If set to true, add the header regardless of the response status code**. Default is false. See the [add_header](http://nginx.org/en/docs/http/ngx_http_headers_module.html#add_header) directive for more information. | ``bool`` | No | +{{}} + +\* -- Supported NGINX variables: `$request_uri`, `$request_method`, `$request_body`, `$scheme`, `$http_`, `$args`, `$arg_`, `$cookie_`, `$host`, `$request_time`, `$request_length`, `$nginx_version`, `$pid`, `$connection`, `$remote_addr`, `$remote_port`, `$time_iso8601`, `$time_local`, `$server_addr`, `$server_port`, `$server_name`, `$server_protocol`, `$connections_active`, `$connections_reading`, `$connections_writing`, `$connections_waiting`, `$ssl_cipher`, `$ssl_ciphers`, `$ssl_client_cert`, `$ssl_client_escaped_cert`, `$ssl_client_fingerprint`, `$ssl_client_i_dn`, `$ssl_client_i_dn_legacy`, `$ssl_client_raw_cert`, `$ssl_client_s_dn`, `$ssl_client_s_dn_legacy`, `$ssl_client_serial`, `$ssl_client_v_end`, `$ssl_client_v_remain`, `$ssl_client_v_start`, `$ssl_client_verify`, `$ssl_curves`, `$ssl_early_data`, `$ssl_protocol`, `$ssl_server_name`, `$ssl_session_id`, `$ssl_session_reused`, `$jwt_claim_` (NGINX Plus only) and `$jwt_header_` (NGINX Plus only). + +{{< note >}} If `always` is false, the response header is added only if the response status code is any of `200`, `201`, `204`, `206`, `301`, `302`, `303`, `304`, `307` or `308`. {{< /note >}} + +### Split + +The split defines a weight for an action as part of the splits configuration. + +In the example below NGINX passes 80% of requests to the upstream `coffee-v1` and the remaining 20% to `coffee-v2`: + +```yaml +splits: +- weight: 80 + action: + pass: coffee-v1 +- weight: 20 + action: + pass: coffee-v2 +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``weight`` | The weight of an action. Must fall into the range ``0..100``. The sum of the weights of all splits must be equal to ``100``. | ``int`` | Yes | +|``action`` | The action to perform for a request. | [action](#action) | Yes | +{{}} + +### Match + +The match defines a match between conditions and an action or splits. + +In the example below, NGINX routes requests with the path `/coffee` to different upstreams based on the value of the cookie `user`: + +- `user=john` -> `coffee-future` +- `user=bob` -> `coffee-deprecated` +- If the cookie is not set or not equal to either `john` or `bob`, NGINX routes to `coffee-stable` + +```yaml +path: /coffee +matches: +- conditions: + - cookie: user + value: john + action: + pass: coffee-future +- conditions: + - cookie: user + value: bob + action: + pass: coffee-deprecated +action: + pass: coffee-stable +``` + +In the next example, NGINX routes requests based on the value of the built-in [`$request_method` variable](https://nginx.org/en/docs/http/ngx_http_core_module.html#var_request_method), which represents the HTTP method of a request: + +- all POST requests -> `coffee-post` +- all non-POST requests -> `coffee` + +```yaml +path: /coffee +matches: +- conditions: + - variable: $request_method + value: POST + action: + pass: coffee-post +action: + pass: coffee +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``conditions`` | A list of conditions. Must include at least 1 condition. | [[]condition](#condition) | Yes | +|``action`` | The action to perform for a request. | [action](#action) | No | +|``splits`` | The splits configuration for traffic splitting. Must include at least 2 splits. | [[]split](#split) | No | +{{}} + +{{< note >}} A match must include exactly one of the following: `action` or `splits`. {{< /note >}} + +### Condition + +The condition defines a condition in a match. + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``header`` | The name of a header. Must consist of alphanumeric characters or ``-``. | ``string`` | No | +|``cookie`` | The name of a cookie. Must consist of alphanumeric characters or ``_``. | ``string`` | No | +|``argument`` | The name of an argument. Must consist of alphanumeric characters or ``_``. | ``string`` | No | +|``variable`` | The name of an NGINX variable. Must start with ``$``. See the list of the supported variables below the table. | ``string`` | No | +|``value`` | The value to match the condition against. How to define a value is shown below the table. | ``string`` | Yes | +{{}} + +{{< note >}} a condition must include exactly one of the following: `header`, `cookie`, `argument` or `variable`. {{< /note >}} + +Supported NGINX variables: `$args`, `$http2`, `$https`, `$remote_addr`, `$remote_port`, `$query_string`, `$request`, `$request_body`, `$request_uri`, `$request_method`, `$scheme`. Find the documentation for each variable [here](https://nginx.org/en/docs/varindex.html). + +The value supports two kinds of matching: + +- *Case-insensitive string comparison*. For example: + - `john` -- case-insensitive matching that succeeds for strings, such as `john`, `John`, `JOHN`. + - `!john` -- negation of the case-insensitive matching for john that succeeds for strings, such as `bob`, `anything`, `''` (empty string). +- *Matching with a regular expression*. Note that NGINX supports regular expressions compatible with those used by the Perl programming language (PCRE). For example: + - `~^yes` -- a case-sensitive regular expression that matches any string that starts with `yes`. For example: `yes`, `yes123`. + - `!~^yes` -- negation of the previous regular expression that succeeds for strings like `YES`, `Yes123`, `noyes`. (The negation mechanism is not part of the PCRE syntax). + - `~*no$` -- a case-insensitive regular expression that matches any string that ends with `no`. For example: `no`, `123no`, `123NO`. + +{{< note >}} A value must not include any unescaped double quotes (`"`) and must not end with an unescaped backslash (`\`). For example, the following are invalid values: `some"value`, `somevalue\`. {{< /note >}} + +### ErrorPage + +The errorPage defines a custom response for a route for the case when either an upstream server responds with (or NGINX generates) an error status code. The custom response can be a redirect or a canned response. See the [error_page](https://nginx.org/en/docs/http/ngx_http_core_module.html#error_page) directive for more information. + +```yaml +path: /coffee +errorPages: +- codes: [502, 503] + redirect: + code: 301 + url: https://nginx.org +- codes: [404] + return: + code: 200 + body: "Original resource not found, but success!" +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``codes`` | A list of error status codes. | ``[]int`` | Yes | +|``redirect`` | The redirect action for the given status codes. | [errorPage.Redirect](#errorpageredirect) | No | +|``return`` | The canned response action for the given status codes. | [errorPage.Return](#errorpagereturn) | No | +{{}} + +{{< note >}} An errorPage must include exactly one of the following: `return` or `redirect`. {{< /note >}} + +### ErrorPage.Redirect + +The redirect defines a redirect for an errorPage. + +In the example below, NGINX responds with a redirect when a response from an upstream server has a 404 status code. + +```yaml +codes: [404] +redirect: + code: 301 + url: ${scheme}://cafe.example.com/error.html +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``code`` | The status code of a redirect. The allowed values are: ``301`` , ``302`` , ``307`` , ``308``. The default is ``301``. | ``int`` | No | +|``url`` | The URL to redirect the request to. Supported NGINX variables: ``$scheme`` and ``$http_x_forwarded_proto``. Variables must be enclosed in curly braces. For example: ``${scheme}``. | ``string`` | Yes | +{{}} + +### ErrorPage.Return + +The return defines a canned response for an errorPage. + +In the example below, NGINX responds with a canned response when a response from an upstream server has either 401 or 403 status code. + +```yaml +codes: [401, 403] +return: + code: 200 + type: application/json + body: | + {\"msg\": \"You don't have permission to do this\"} + headers: + - name: x-debug-original-statuses + value: ${upstream_status} +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``code`` | The status code of the response. The default is the status code of the original response. | ``int`` | No | +|``type`` | The MIME type of the response. The default is ``text/html``. | ``string`` | No | +|``body`` | The body of the response. Supported NGINX variable: ``$upstream_status`` . Variables must be enclosed in curly braces. For example: ``${upstream_status}``. | ``string`` | Yes | +|``headers`` | The custom headers of the response. | [[]errorPage.Return.Header](#errorpagereturnheader) | No | +{{}} + +### ErrorPage.Return.Header + +The header defines an HTTP Header for a canned response in an errorPage: + +```yaml +name: x-debug-original-statuses +value: ${upstream_status} +``` + +{{}} +|Field | Description | Type | Required | +| ---| ---| ---| --- | +|``name`` | The name of the header. | ``string`` | Yes | +|``value`` | The value of the header. Supported NGINX variable: ``$upstream_status`` . Variables must be enclosed in curly braces. For example: ``${upstream_status}``. | ``string`` | No | +{{}} + +## Using VirtualServer and VirtualServerRoute + +You can use the usual `kubectl` commands to work with VirtualServer and VirtualServerRoute resources, similar to Ingress resources. + +For example, the following command creates a VirtualServer resource defined in `cafe-virtual-server.yaml` with the name `cafe`: + +```shell +kubectl apply -f cafe-virtual-server.yaml +``` +```text +virtualserver.k8s.nginx.org "cafe" created +``` + +You can get the resource by running: + +```shell +kubectl get virtualserver cafe +``` +```text +NAME STATE HOST IP PORTS AGE +cafe Valid cafe.example.com 12.13.23.123 [80,443] 3m +``` + +In `kubectl get` and similar commands, you can use the short name `vs` instead of `virtualserver`. + +Similarly, for VirtualServerRoute you can use `virtualserverroute` or the short name `vsr`. + +### Using Snippets + +Snippets allow you to insert raw NGINX config into different contexts of NGINX configuration. In the example below, we use snippets to configure several NGINX features in a VirtualServer: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: cafe + namespace: cafe +spec: + http-snippets: | + limit_req_zone $binary_remote_addr zone=mylimit:10m rate=1r/s; + proxy_cache_path /tmp keys_zone=one:10m; + host: cafe.example.com + tls: + secret: cafe-secret + server-snippets: | + limit_req zone=mylimit burst=20; + upstreams: + - name: tea + service: tea-svc + port: 80 + - name: coffee + service: coffee-svc + port: 80 + routes: + - path: /tea + location-snippets: | + proxy_cache one; + proxy_cache_valid 200 10m; + action: + pass: tea + - path: /coffee + action: + pass: coffee +``` + +For additional information, view the [Advanced configuration with Snippets]({{< ref "/nic/configuration/ingress-resources/advanced-configuration-with-snippets.md" >}}) topic. + +### Validation + +Two types of validation are available for VirtualServer and VirtualServerRoute resources: + +- *Structural validation* by the `kubectl` and Kubernetes API server. +- *Comprehensive validation* by NGINX Ingress Controller. + +#### Structural Validation + +The custom resource definitions for VirtualServer and VirtualServerRoute include structural OpenAPI schema which describes the type of every field of those resources. + +If you try to create (or update) a resource that violates the structural schema (for example, you use a string value for the port field of an upstream), `kubectl` and Kubernetes API server will reject such a resource: + +- Example of `kubectl` validation: + + ```shell + kubectl apply -f cafe-virtual-server.yaml + ``` + ```text + error: error validating "cafe-virtual-server.yaml": error validating data: ValidationError(VirtualServer.spec.upstreams[0].port): invalid type for org.nginx.k8s.v1.VirtualServer.spec.upstreams.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + ``` + +- Example of Kubernetes API server validation: + + ```shell + kubectl apply -f cafe-virtual-server.yaml --validate=false + ``` + ```text + The VirtualServer "cafe" is invalid: []: Invalid value: map[string]interface {}{ ... }: validation failure list: + spec.upstreams.port in body must be of type integer: "string" + ``` + +If a resource is not rejected (it doesn't violate the structural schema), NGINX Ingress Controller will validate it further. + +#### Comprehensive Validation + +NGINX Ingress Controller validates the fields of the VirtualServer and VirtualServerRoute resources. If a resource is invalid, NGINX Ingress Controller will reject it: the resource will continue to exist in the cluster, but NGINX Ingress Controller will ignore it. + +You can check if NGINX Ingress Controller successfully applied the configuration for a VirtualServer. For our example `cafe` VirtualServer, we can run: + +```shell +kubectl describe vs cafe +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 16s nginx-ingress-controller Configuration for default/cafe was added or updated +``` + +Note how the events section includes a Normal event with the AddedOrUpdated reason that informs us that the configuration was successfully applied. + +If you create an invalid resource, NGINX Ingress Controller will reject it and emit a Rejected event. For example, if you create a VirtualServer `cafe` with two upstream with the same name `tea`, you will get: + +```shell +kubectl describe vs cafe +``` +```text +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Rejected 12s nginx-ingress-controller VirtualServer default/cafe is invalid and was rejected: spec.upstreams[1].name: Duplicate value: "tea" +``` + +Note how the events section includes a Warning event with the Rejected reason. + +Additionally, this information is also available in the `status` field of the VirtualServer resource. Note the Status section of the VirtualServer: + +```shell +kubectl describe vs cafe +``` +```text +... +Status: + External Endpoints: + Ip: 12.13.23.123 + Ports: [80,443] + Message: VirtualServer default/cafe is invalid and was rejected: spec.upstreams[1].name: Duplicate value: "tea" + Reason: Rejected + State: Invalid +``` + +NGINX Ingress Controller validates VirtualServerRoute resources in a similar way. + +**Note**: If you make an existing resource invalid, NGINX Ingress Controller will reject it and remove the corresponding configuration from NGINX. + +## Customization using ConfigMap + +You can customize the NGINX configuration for VirtualServer and VirtualServerRoutes resources using the [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource). Most of the ConfigMap keys are supported, with the following exceptions: + +- `proxy-hide-headers` +- `proxy-pass-headers` +- `hsts` +- `hsts-max-age` +- `hsts-include-subdomains` +- `hsts-behind-proxy` +- `redirect-to-https` +- `ssl-redirect` diff --git a/content/nic/glossary.md b/content/nic/glossary.md new file mode 100644 index 000000000..d76f9e3db --- /dev/null +++ b/content/nic/glossary.md @@ -0,0 +1,30 @@ +--- +description: null +docs: DOCS-1446 +title: Glossary +weight: 10000 +--- + +This is a glossary of terms related to F5 NGINX Ingress Controller and Kubernetes as a whole. + +--- + +## Ingress {#ingress} + +_Ingress_ refers to an _Ingress Resource_, a Kubernetes API object which allows access to [Services](https://kubernetes.io/docs/concepts/services-networking/service/) within a cluster. They are managed by an [Ingress Controller]({{< ref "/nic/glossary.md#ingress-controller">}}). + +_Ingress_ resources enable the following functionality: + +- **Load balancing**, extended through the use of Services +- **Content-based routing**, using hosts and paths +- **TLS/SSL termination**, based on hostnames + +For additional information, please read the official [Kubernetes Ingress Documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/). + +--- + +## Ingress Controller {#ingress-controller} + +*Ingress Controllers* are applications within a Kubernetes cluster that enable [Ingress]({{< ref "/nic/glossary.md#ingress">}}) resources to function. They are not automatically deployed with a Kubernetes cluster, and can vary in implementation based on intended use, such as load balancing algorithms for Ingress resources. + +[The design of NGINX Ingress Controller]({{< ref "/nic/overview/design.md">}}) explains the technical details of NGINX Ingress Controller. diff --git a/content/nic/installation/_index.md b/content/nic/installation/_index.md new file mode 100644 index 000000000..4ed49bfcb --- /dev/null +++ b/content/nic/installation/_index.md @@ -0,0 +1,8 @@ +--- +title: Installation +description: +weight: 400 +menu: + docs: + parent: NGINX Ingress Controller +--- diff --git a/content/nic/installation/build-nginx-ingress-controller.md b/content/nic/installation/build-nginx-ingress-controller.md new file mode 100644 index 000000000..4f37d8f25 --- /dev/null +++ b/content/nic/installation/build-nginx-ingress-controller.md @@ -0,0 +1,203 @@ +--- +title: Build NGINX Ingress Controller +toc: true +weight: 400 +type: how-to +product: NIC +docs: DOCS-1453 +--- + +This document describes how to build an F5 NGINX Ingress Controller image from source code and upload it to a private Docker registry. + +It also includes information on the Makefile targets and variables. + +{{}} If you do not need to build a custom image, see the [pre-built image options](#pre-built-images) at the end of this guide. {{}} + +## Before you begin + +To get started, you need the following software installed on your machine: + +- [Docker v19.03 or higher](https://docs.docker.com/engine/release-notes/19.03/) +- [GNU Make](https://www.gnu.org/software/make/) +- [git](https://git-scm.com/) +- [OpenSSL](https://www.openssl.org/), optionally, if you would like to generate a self-signed certificate and a key for the default server. +- For NGINX Plus users, download the certificate (_nginx-repo.crt_) and key (_nginx-repo.key_) from [MyF5](https://my.f5.com). + +Although NGINX Ingress Controller is written in Golang, you don't need to have Golang installed. + +You can download the precompiled binary file or build NGINX Ingress Controller in a Docker container. + +--- + +## Prepare the environment {#prepare-environment} + +Get your system ready for building and pushing the NGINX Ingress Controller image. + +1. Sign in to your private registry. Replace `` with the path to your own private registry. + + ```shell + docker login + ``` + +2. Clone the NGINX Ingress Controller GitHub repository. Replace `` with the version of NGINX Ingress Controller you want. + + ```shell + git clone https://github.com/nginx/kubernetes-ingress.git --branch + cd kubernetes-ingress + ``` + + For instance if you want to clone version v{{< nic-version >}}, the commands to run would be: + + ```shell + git clone https://github.com/nginx/kubernetes-ingress.git --branch v{{< nic-version >}} + cd kubernetes-ingress + ``` + +--- + +## Build the NGINX Ingress Controller image {#build-image} + +After setting up your environment, follow these steps to build the NGINX Ingress Controller image. + +{{< note >}} If you have a local Golang environment and want to build the binary yourself, remove `TARGET=download` from the make commands. If you don't have Golang but still want to build the binary, use `TARGET=container`. {{< /note >}} + +### For NGINX + +Build the image. Replace `` with your private registry's path. + +- For a Debian-based image: + + ```shell + make debian-image PREFIX=/nginx-ingress TARGET=download + ``` + +- For an Alpine-based image: + + ```shell + make alpine-image PREFIX=/nginx-ingress TARGET=download + ``` + +**What to expect**: The image is built and tagged with a version number, which is derived from the `VERSION` variable in the [_Makefile_](#makefile-details). This version number is used for tracking and deployment purposes. + +### For NGINX Plus + +Place your NGINX Plus license files (_nginx-repo.crt_ and _nginx-repo.key_) in the project's root folder. To verify they're in place, run: + +```shell +ls nginx-repo.* +``` + +You should see: + +```text +nginx-repo.crt nginx-repo.key +``` + +Build the image. Replace `` with your private registry's path. + +```shell +make debian-image-plus PREFIX=/nginx-plus-ingress TARGET=download +``` + +
+ +**What to expect**: The image is built and tagged with a version number, which is derived from the `VERSION` variable in the [_Makefile_](#makefile-details). This version number is used for tracking and deployment purposes. + +{{}} If a patch for NGINX Plus is released, make sure to rebuild your image to get the latest version. If your system is caching the Docker layers and not updating the packages, add `DOCKER_BUILD_OPTIONS="--pull --no-cache"` to the make command. {{}} + +--- + +## Push the image to your private registry {#push-image} + +Once you've successfully built the NGINX or NGINX Plus Ingress Controller image, the next step is to upload it to your private Docker registry. This makes the image available for deployment to your Kubernetes cluster. + +### For NGINX + +Upload the NGINX image. If you're using a custom tag, append `TAG=your-tag` to the command. Replace `` with your private registry's path. + +```shell +make push PREFIX=/nginx-ingress +``` + +### For NGINX Plus + +Upload the NGINX Plus image. Like with the NGINX image, if you're using a custom tag, add `TAG=your-tag` to the end of the command. Replace `` with your private registry's path. + +```shell +make push PREFIX=/nginx-plus-ingress +``` + +--- + +## Makefile details {#makefile-details} + +This section provides comprehensive information on the targets and variables available in the _Makefile_. These targets and variables allow you to customize how you build, tag, and push your NGINX or NGINX Plus images. + +### Key Makefile targets {#key-makefile-targets} + +{{}}To view available _Makefile_ targets, run `make` with no target or type `make help`.{{}} + +Key targets include: + +{{}} +|
Target | Description | +|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| _build_ | Creates the NGINX Ingress Controller binary with your local Go environment. | +| _alpine-image_ | Builds an Alpine-based image with NGINX. | +| _alpine-image-plus_ | Builds an Alpine-based image with NGINX Plus. | +| _alpine-image-plus-fips_ | Builds an Alpine-based image with NGINX Plus and FIPS. | +| _alpine-image-nap-v5-plus-fips_ | Builds an Alpine-based image with NGINX Plus, the [NGINX App Protect WAF v5](/nginx-app-protect/) module and FIPS. | +| _debian-image_ | Builds a Debian-based image with NGINX. | +| _debian-image-plus_ | Builds a Debian-based image with NGINX Plus. | +| _debian-image-nap-plus_ | Builds a Debian-based image with NGINX Plus and the [NGINX App Protect WAF](/nginx-app-protect/) module. | +| _debian-image-nap-v5-plus_ | Builds a Debian-based image with NGINX Plus and the [NGINX App Protect WAF v5](/nginx-app-protect/) module. | +| _debian-image-dos-plus_ | Builds a Debian-based image with NGINX Plus and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module. | +| _debian-image-nap-dos-plus_ | Builds a Debian-based image with NGINX Plus, [NGINX App Protect WAF](/nginx-app-protect/) and [NGINX App Protect DoS](/nginx-app-protect-dos/) modules. | +| _ubi-image_ | Builds a UBI-based image with NGINX for [OpenShift](https://www.openshift.com/) clusters. | +| _ubi-image-plus_ | Builds a UBI-based image with NGINX Plus for [OpenShift](https://www.openshift.com/) clusters. | +| _ubi-image-nap-plus_ | Builds a UBI-based image with NGINX Plus and the [NGINX App Protect WAF](/nginx-app-protect/) module for [OpenShift](https://www.openshift.com/) clusters. | +| _ubi-image-nap-v5-plus_ | Builds a UBI-based image with NGINX Plus and the [NGINX App Protect WAF v5](/nginx-app-protect/) module for [OpenShift](https://www.openshift.com/) clusters. | +| _ubi-image-dos-plus_ | Builds a UBI-based image with NGINX Plus and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module for [OpenShift](https://www.openshift.com/) clusters. | +| _ubi-image-nap-dos-plus_ |

Builds a UBI-based image with NGINX Plus, [NGINX App Protect WAF](/nginx-app-protect/) and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module for [OpenShift](https://www.openshift.com/) clusters.

**Important**: Save your RHEL organization and activation keys in a file named _rhel_license_ at the project root.

For instance:

RHEL_ORGANIZATION=1111111
RHEL_ACTIVATION_KEY=your-key
| +{{}} + +--- + +### Additional useful targets {#other-makefile-targets} + +A few other useful targets: + +{{}} +|
Target
| Description | +|---------------------------------------|---------------| +| _push_ | Pushes the built image to the Docker registry. Configures with `PREFIX` and `TAG`. | +| _all_ | Runs `test`, `lint`, `verify-codegen`, `update-crds`, and `debian-image`. Stops and reports an error if any of these targets fail. | +| _test_ | Runs unit tests. | +{{}} + +### Makefile variables you can customize {#makefile-variables} + +The _Makefile_ includes several key variables. You have the option to either modify these variables directly in the _Makefile_ or override them when you run the `make` command. + +{{}} +|
Variable
| Description | +|-----------------------------------------|---------------| +| _ARCH_ | Defines the architecture for the image and binary. The default is `amd64`, but you can also use `arm64`. | +| _PREFIX_ | Gives the image its name. The default is `nginx/nginx-ingress`. | +| _TAG_ | Adds a tag to the image. This is often the version of NGINX Ingress Controller. | +| _DOCKER\_BUILD\_OPTIONS_ | Allows for additional [options](https://docs.docker.com/engine/reference/commandline/build/#options) during the `docker build` process, like `--pull`. | +| _TARGET_ |

Determines the build environment. NGINX Ingress Controller compiles locally in a Golang environment by default. Ensure the NGINX Ingress Controller repo resides in your `$GOPATH` if you select this option.

Alternatively, you can set `TARGET=container` to build using a Docker [Golang](https://hub.docker.com/_/golang/) container. To skip compiling the binary if you're on a specific tag or the latest `main` branch commit, set `TARGET=download`.

| +{{}} + +--- + +## Alternatives to building your own image {#pre-built-images} + +If you prefer not to build your own NGINX Ingress Controller image, you can use pre-built images. Here are your options: + +**NGINX Ingress Controller**: Download the image `nginx/nginx-ingress` from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress) or [GitHub](https://github.com/nginx/kubernetes-ingress/pkgs/container/kubernetes-ingress). + +**NGINX Plus Ingress Controller**: You have two options for this: + +- Download the image using your NGINX Ingress Controller subscription certificate and key. View the [Get NGINX Ingress Controller from the F5 Registry]({{< ref "/nic/installation/nic-images/get-registry-image" >}}) topic. +- Use your NGINX Ingress Controller subscription JWT token to get the image. View the [Get the NGINX Ingress Controller image with JWT]({{< ref "/nic/installation/nic-images/get-image-using-jwt.md" >}}) topic. diff --git a/content/nic/installation/create-license-secret.md b/content/nic/installation/create-license-secret.md new file mode 100644 index 000000000..91be0f9f9 --- /dev/null +++ b/content/nic/installation/create-license-secret.md @@ -0,0 +1,155 @@ +--- +title: Create a license Secret +toc: true +weight: 300 +nd-content-type: how-to +nd-product: NIC +--- + +This document explains how to create and use a license secret for F5 NGINX Ingress Controller. + +# Overview + +NGINX Plus Ingress Controller requires a valid JSON Web Token (JWT) to download the container image from the F5 registry. From version 4.0.0, this JWT token is also required to run NGINX Plus. + +This requirement is part of F5’s broader licensing program and aligns with industry best practices. The JWT will streamline subscription renewals and usage reporting, helping you manage your NGINX Plus subscription more efficiently. The [telemetry](#telemetry) data we collect helps us improve our products and services to better meet your needs. + +The JWT is required for validating your subscription and reporting telemetry data. For environments connected to the internet, telemetry is automatically sent to F5’s licensing endpoint. In offline environments, telemetry is routed through [NGINX Instance Manager]({{< ref "/nim/" >}}). By default usage is reported every hour and also whenever NGINX is reloaded. + +{{< note >}} Read the [subscription licenses topic]({{< ref "/solutions/about-subscription-licenses.md#for-internet-connected-environments" >}}) for a list of IPs associated with F5's licensing endpoint (`product.connect.nginx.com`). {{}} + +## Set up your NGINX Plus license + +### Download the JWT + +{{< include "/nic/installation/download-jwt.md" >}} + +### Create the Secret + +The JWT needs to be configured before deploying NGINX Ingress Controller. The JWT will be stored in a Kubernetes Secret of type `nginx.com/license`, and can be created with the following command. + +```shell +kubectl create secret generic license-token --from-file=license.jwt= --type=nginx.com/license -n +``` +You can now delete the downloaded `.jwt` file. + +{{< note >}} +The Secret needs to be in the same Namespace as the NGINX Ingress Controller Pod(s). +{{}} + +{{< include "/nic/installation/jwt-password-note.md" >}} + +### Use the NGINX Plus license Secret + +If using a name other than the default `license-token`, provide the name of this Secret when installing NGINX Ingress Controller: + +{{}} + +{{%tab name="Helm"%}} + +Specify the Secret name using the `controller.mgmt.licenseTokenSecretName` Helm value. + +For detailed guidance on creating the Management block via Helm, refer to the [Helm configuration documentation]({{< ref "/nic/installation/installing-nic/installation-with-helm/#configuration" >}}). + +{{% /tab %}} + +{{%tab name="Manifests"%}} + +Specify the Secret name in the `license-token-secret-name` Management ConfigMap key. + +For detailed guidance on creating the Management ConfigMap, refer to the [Management ConfigMap Resource Documentation]({{< ref "/nic/configuration/global-configuration/mgmt-configmap-resource/" >}}). + +{{% /tab %}} + +{{}} + +If you are reporting to the default licensing endpoint, then you can now proceed with [installing NGINX Ingress Controller]({{< ref "nic/installation/installing-nic/" >}}). Otherwise, follow the steps below to configure reporting to NGINX Instance Manager + +### Create report for NGINX Instance Manager {#nim} + +If you are deploying NGINX Ingress Controller in an "air-gapped" environment you will need to report to [NGINX Instance Manager]({{< ref "/nim/" >}}) instead of the default licensing endpoint. + +First, you must specify the endpoint of your NGINX Instance Manager. + +{{}} + +{{%tab name="Helm"%}} + +Specify the endpoint using the `controller.mgmt.usageReport.endpoint` helm value. + +{{% /tab %}} + +{{%tab name="Manifests"%}} + +Specify the endpoint in the `usage-report-endpoint` Management ConfigMap key. + +{{% /tab %}} + +{{}} + +#### Configure SSL certificates and SSL trusted certificates {#nim-cert} + +To configure SSL certificates or SSL trusted certificates, extra steps are necessary. + +To use Client Auth with NGINX Instance Manager, first create a Secret of type `kubernetes.io/tls` in the same namespace as the NGINX Ingress Controller pods. + +```shell +kubectl create secret tls ssl-certificate --cert= --key= -n +``` + +To provide a SSL trusted certificate, and an optional Certificate Revocation List, create a Secret of type `nginx.org/ca` in the Namespace that the NIC Pod(s) are in. + +```shell +kubectl create secret generic ssl-trusted-certificate \ + --from-file=ca.crt= \ + --from-file=ca.crl= \ # optional + --type=nginx.org/ca +``` + +Providing an optional CRL (certificate revocation list) will configure the [`ssl_crl`](https://nginx.org/en/docs/ngx_mgmt_module.html#ssl_crl) directive. + +{{}} + +{{%tab name="Helm"%}} + +Specify the SSL certificate Secret name using the `controller.mgmt.sslCertificateSecretName` Helm value. + +Specify the SSL trusted certificate Secret name using the `controller.mgmt.sslTrustedCertificateSecretName` Helm value. + +{{% /tab %}} + +{{%tab name="Manifests"%}} + +Specify the SSL certificate Secret name in the `ssl-certificate-secret-name` management ConfigMap key. + +Specify the SSL trusted certificate Secret name in the `ssl-trusted-certificate-secret-name` management ConfigMap key. + +{{% /tab %}} + +{{}} + +
+ +Once these Secrets are created and configured, you can now [install NGINX Ingress Controller ]({{< ref "/nic/installation/installing-nic/" >}}). + + +## What’s reported and how it’s protected {#telemetry} + +NGINX Plus reports the following data every hour by default: + +- **NGINX version and status**: The version of NGINX Plus running on the instance. +- **Instance UUID**: A unique identifier for each NGINX Plus instance. +- **Traffic data**: + - **Bytes received from and sent to clients**: HTTP and stream traffic volume between clients and NGINX Plus. + - **Bytes received from and sent to upstreams**: HTTP and stream traffic volume between NGINX Plus and upstream servers. + - **Client connections**: The number of accepted client connections (HTTP and stream traffic). + - **Requests handled**: The total number of HTTP requests processed. +- **NGINX uptime**: The number of reloads and worker connections during uptime. +- **Usage report timestamps**: Start and end times for each usage report. +- **Kubernetes node details**: Information about Kubernetes nodes. + +### Security and privacy of reported data + +All communication between your NGINX Plus instances, NGINX Instance Manager, and F5’s licensing endpoint (`product.connect.nginx.com`) is protected using **SSL/TLS** encryption. + +Only **operational metrics** are reported — no **personally identifiable information (PII)** or **sensitive customer data** is transmitted. diff --git a/content/nic/installation/ingress-nginx.md b/content/nic/installation/ingress-nginx.md new file mode 100644 index 000000000..cdace0791 --- /dev/null +++ b/content/nic/installation/ingress-nginx.md @@ -0,0 +1,546 @@ +--- +docs: DOCS-1469 +doctypes: +- tutorial +tags: +- docs +title: Migrate from Ingress-NGINX Controller to NGINX Ingress Controller +toc: true +weight: 500 +--- + +This document describes how to migrate from the community-maintained Ingress-NGINX Controller to F5 NGINX Ingress Controller. + +## Overview + +This page explains two different ways to migrate from the community-maintained [Ingress-NGINX Controller](https://github.com/kubernetes/ingress-nginx) project to NGINX Ingress Controller: using NGINX's Ingress Resources or with Kubernetes's built-in Ingress Resources. This is typically because of implementation differences, and to take advantage of features such as [NGINX Plus integration]({{}}). + +The information in this guide is extracted from a free eBook called "_Kubernetes Ingress Controller Deployment and Security with NGINX_", which can be downloaded from the [NGINX Library](https://www.nginx.com/resources/library/kubernetes-ingress-controller-deployment-security-nginx/). + +## Before you begin + +To complete the instructions in this guide, you need the following: + +- A working knowledge of [Ingress Controllers]({{}}). +- An [NGINX Ingress Controller installation]({{}}) on the same host as an existing Ingress-NGINX Controller. + +There are two primary paths for migrating between the community Ingress-NGINX Controller to NGINX Ingress Controller: + +- Using NGINX Ingress Resources +- Using Kubernetes Ingress Resources. + +## Migration with NGINX Ingress resources +This path uses Kubernetes Ingress Resources to set root permissions, then NGINX Ingress Resources for configuration using custom resource definitions (CRDs): + +* [VirtualServer and VirtualServerRoute]({{}}) +* [TransportServer]({{}}) +* [GlobalConfiguration]({{}}) +* [Policy]({{}}) + +### Configuring SSL termination and HTTP path-based routing +The following two code examples correspond to a Kubernetes Ingress Resource and an [NGINX VirtualServer Resource]({{}}). Although the syntax and indentation is different, they accomplish the same basic Ingress functions, used for SSL termination and Layer 7 path-based routing. + +**Kubernetes Ingress Resource** +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nginx-test +spec: + tls: + - hosts: + - foo.bar.com + secretName: tls-secret + rules: + - host: foo.bar.com + http: + paths: + - path: /login + backend: + serviceName: login-svc + servicePort: 80 + - path: /billing + serviceName: billing-svc + servicePort: 80 +``` + +**NGINX VirtualServer Resource** +```yaml +apiVersion: networking.k8s.io/v1 +kind: VirtualServer +metadata: + name: nginx-test +spec: + host: foo.bar.com + tls: + secret: tls-secret + upstreams: + - name: login + service: login-svc + port: 80 + - name: billing + service: billing-svc + port: 80 + routes: + - path: /login + action: + pass: login + - path: /billing + action: + pass: billing +``` + +### Configuring TCP/UDP load balancing and TLS passthrough +NGINX Ingress Controller exposes TCP and UDP services using [TransportServer]({{}}) and [GlobalConfiguration]({{}}) resources. These resources provide a broad range of options for TCP/UDP and TLS Passthrough load balancing. By contrast, the community Ingress-NGINX Controller exposes TCP/UDP services by using a Kubernetes ConfigMap object. + +--- + +### Convert Ingress-NGINX Controller annotations to NGINX Ingress resources +Kubernetes deployments often need to extend basic Ingress rules for advanced use cases such as canary and blue-green deployments, traffic throttling, and ingress-egress traffic manipulation. The community Ingress-NGINX Controller implements many of these using Kubernetes annotations with custom Lua extensions. + +These custom Lua extensions are intended for specific NGINX Ingress resource definitions and may not be as granular as required for advanced use cases. The following examples show how to convert these annotations into NGINX Ingress Controller Resources. + +--- + +#### Canary deployments +Canary and blue-green deployments allow you to push code changes to production environments without disrupting existing users. NGINX Ingress Controller runs them on the data plane: to migrate from the community Ingress-NGINX Controller, you must map the latter's annotations to [VirtualServer and VirtualServerRoute resources]({{}}). + +The Ingress-NGINX Controller evaluates canary annotations in the following order: + +1. _nginx.ingress.kubernetes.io/canary-by-header_ +1. _nginx.ingress.kubernetes.io/canary-by-cookie_ +1. _nginx.ingress.kubernetes.io/canary-by-weight_ + +For NGINX Ingress Controller to evalute them the same way, they must appear in the same order in the VirtualServer or VirtualServerRoute Manifest. + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/canary: "true" +nginx.ingress.kubernetes.io/canary-by-header: "httpHeader" +``` + +**NGINX Ingress Controller** +```yaml +matches: +- conditions: + - header: httpHeader + value: never + action: + pass: echo + - header: httpHeader + value: always + action: + pass: echo-canary +action: + pass: echo +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/canary: "true" +nginx.ingress.kubernetes.io/canary-by-header: "httpHeader" +nginx.ingress.kubernetes.io/canary-by-header-value: "my-value" +``` + +**NGINX Ingress Controller** +```yaml +matches: +- conditions: + - header: httpHeader + value: my-value + action: + pass: echo-canary +action: + pass: echo +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/canary: "true" +nginx.ingress.kubernetes.io/canary-by-cookie: "cookieName" +``` + +**NGINX Ingress Controller** +```yaml +matches: +- conditions: + - cookie: cookieName + value: never + action: + pass: echo + - cookie: cookieName + value: always + action: + pass: echo-canary +action: + pass: echo +``` + +--- + +#### Traffic control +Environments using microservices tend to use extensive traffic-control policies to manage ephemeral applications using circuit breaking and rate and connection limiting to prevent error conditions due to unhealthy states or abnormal behavior. + +The following examples map Ingress-NGINX Controller annotations to NGINX [VirtualServer and VirtualServerRoute resources]({{}}) for rate limiting, custom HTTP errors, custom default backend and URI rewriting. + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/custom-http-errors: "code" + +nginx.ingress.kubernetes.io/default-backend: "default-svc" +``` + +**NGINX Ingress Controller** +```yaml +errorPages: +- codes: [code] + redirect: + code: 301 + url: default-svc +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-connections: "number" +``` + +**NGINX Ingress Controller** +```yaml +http-snippets: | + limit_conn_zone $binary_remote_addr zone=zone_name:size; +routes: +- path: /path + location-snippets: | + limit_conn zone_name number; +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-rate: "number" +nginx.ingress.kubernetes.io/limit-rate-after: "number" +``` + +**NGINX Ingress Controller** +```yaml +location-snippets: | + limit_rate number; + + limit_rate_after number; +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-rpm: "number" +nginx.ingress.kubernetes.io/limit-burst-multiplier: "multiplier" +``` + +**NGINX Ingress Controller** +```yaml +rateLimit: + rate: numberr/m + + burst: number * multiplier + key: ${binary_remote_addr} + zoneSize: size +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-rps: "number" +nginx.ingress.kubernetes.io/limit-burst-multiplier: "multiplier" +``` + +**NGINX Ingress Controller** +```yaml +rateLimit: + rate: numberr/s + + burst: number * multiplier + key: ${binary_remote_addr} + zoneSize: size +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-whitelist: "CIDR" +``` + +**NGINX Ingress Controller** +```yaml +http-snippets: | +server-snippets: | +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/rewrite-target: "URI" +``` + +**NGINX Ingress Controller** +```yaml +rewritePath: "URI" +``` + +There are four Ingress-NGINX Controller annotations without NGINX Ingress resource fields yet: they must be handled using snippets. + +- _nginx.ingress.kubernetes.io/limit-connections_ +- _nginx.ingress.kubernetes.io/limit-rate_ +- _nginx.ingress.kubernetes.io/limit-rate-after_ +- _nginx.ingress.kubernetes.io/limit-whitelist_ + +--- + +#### Header manipulation +Manipulating HTTP headers is useful in many cases, as they contain information that is important and relevant to systems involved in HTTP transactions. The community Ingress-NGINX Controller supports enabling and configuring cross-origin resource sharing (CORS) headings used by AJAX applications, where front-end Javascript code interacts with backend applications or web servers. + +These code blocks show how the Ingress-NGINX annotations correspond to NGINX Ingress Controller [VirtualServer and VirtualServerRoute resources]({{}}). + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/enable-cors: "true" +nginx.ingress.kubernetes.io/cors-allow-credentials: "true" + +nginx.ingress.kubernetes.io/cors-allow-headers: "X-Forwarded-For" + +nginx.ingress.kubernetes.io/cors-allow-methods: "PUT, GET, POST, OPTIONS" + +nginx.ingress.kubernetes.io/cors-allow-origin: "*" + +nginx.ingress.kubernetes.io/cors-max-age: "seconds" +``` + +**NGINX Ingress Controller** +```yaml +responseHeaders: + add: + - name: Access-Control-Allow-Credentials + value: "true" + - name: Access-Control-Allow-Headers + value: "X-Forwarded-For" + - name: Access-Control-Allow-Methods + value: "PUT, GET, POST, OPTIONS" + - name: Access-Control-Allow-Origin + value: "*" + - name: Access-Control-Max-Age + value: "seconds" +``` + +--- + +#### Proxying and load balancing +NGINX Ingress Controller has multiple proxy and load balancing functionalities you may want to configure based on the use case, such as configuring the load balancing algorithm and the timeout and buffering settings for proxied connections. + +This table shows how Ingress-NGINX Controller annotations map to statements in the upstream field for [VirtualServer and VirtualServerRoute resources]({{}}), covering load balancing, proxy timeout, proxy buffering and connection routing for a services' ClusterIP address and port. + +{{< bootstrap-table "table table-bordered table-striped table-responsive" >}} +| Ingress-NGINX Controller | NGINX Ingress Controller | +| ------------------------ | ------------------------ | +| _nginx.ingress.kubernetes.io/load-balance_ | _lb-method_ | +| _nginx.ingress.kubernetes.io/proxy-buffering_ | _buffering_ | +| _nginx.ingress.kubernetes.io/proxy-buffers-number_ | _buffers_ | +| _nginx.ingress.kubernetes.io/proxy-buffer-size_ | _buffers_ | +| _nginx.ingress.kubernetes.io/proxy-connect-timeout_ | _connect-timeout_ | +| _nginx.ingress.kubernetes.io/proxy-next-upstream_ | _next-upstream_ | +| _nginx.ingress.kubernetes.io/proxy-next-upstream-timeout_ | _next-upstream-timeout_ | +| _nginx.ingress.kubernetes.io/proxy-read-timeout_ | _read-timeout_ | +| _nginx.ingress.kubernetes.io/proxy-send-timeout_ | _send-timeout_ | +| _nginx.ingress.kubernetes.io/service-upstream_ | _use-cluster-ip_ | +{{% /bootstrap-table %}} + +#### mTLS authentication + +mTLS authentication is a way of enforcing mutual authentication on traffic entering and exiting a cluster (north-sourth traffic). This secure form of communication is common within a service mesh, commonly used in strict zero-trust environments. + +NGINX Ingress Controller layer can handle mTLS authentication for end systems through the presentation of valid certificates for external connections. It accomplishes this through [Policy]({{}}) resources, which correspond to Ingress-NGINX Controller annotations for [client certificate authentication](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#client-certificate-authentication) and [backend certificate authentication](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#backend-certificate-authentication). + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/auth-tls-secret: secretName +nginx.ingress.kubernetes.io/auth-tls-verify-client: "on" +nginx.ingress.kubernetes.io/auth-tls-verify-depth: "1" +``` + +**NGINX Ingress Controller** +```yaml +ingressMTLS: + clientCertSecret: secretName + verifyClient: "on" + + verifyDepth: 1 +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/proxy-ssl-secret: "secretName" +nginx.ingress.kubernetes.io/proxy-ssl-verify: "on|off" +nginx.ingress.kubernetes.io/proxy-ssl-verify-depth: "1" +nginx.ingress.kubernetes.io/proxy-ssl-protocols: "TLSv1.2" +nginx.ingress.kubernetes.io/proxy-ssl-ciphers: "DEFAULT" +nginx.ingress.kubernetes.io/proxy-ssl-name: "server-name" +nginx.ingress.kubernetes.io/proxy-ssl-server-name: "on|off" +``` + +**NGINX Ingress Controller** +```yaml +egressMTLS: + tlsSecret: secretName + + verifyServer: true|false + + verifyDepth: 1 + + protocols: TLSv1.2 + + ciphers: DEFAULT + + sslName: server-name + + serverName: true|false +``` + +--- + +#### Session persistence with NGINX Plus +With [NGINX Plus]({{}}), you can use [Policy]({{}}) resources for session persistence, which have corresponding annotations for the community Ingress-NGINX Controller. + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/affinity: "cookie" +nginx.ingress.kubernetes.io/session-cookie-name: "cookieName" +nginx.ingress.kubernetes.io/session-cookie-expires: "x" +nginx.ingress.kubernetes.io/session-cookie-path: "/route" +nginx.ingress.kubernetes.io/session-cookie-secure: "true" +``` + +**NGINX Ingress Controller** +```yaml +sessionCookie: + enable: true + + name: cookieName + + expires: xh + + path: /route + + secure: true +``` + +## Migration with Kubernetes Ingress resources +The other option for migrating from the community Ingress-NGINX Controller to NGINX Ingress Controller is using only [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) and [ConfigMaps](https://kubernetes.io/docs/concepts/configuration/configmap/) from standard Kubernetes resources, potentially relying on [mergeable Ingress types](https://github.com/nginx/kubernetes-ingress/tree/main/examples/ingress-resources/mergeable-ingress-types). + +This ensures that all configuration is kept in the Ingress object. + +{{< warning >}} +You should avoid altering the `spec` field of the Ingress resource when taking this option. Ingress-NGINX Controller and NGINX Ingress Controller differ slightly in their implementations: changing the Kubernetes Ingress can create incompatibility issues. +{{< /warning >}} + +### Advanced configuration with annotations +This table maps the Ingress-NGINX Controller annotations to NGINX Ingress Controller's equivalent annotations, and the respective NGINX Directive. + +{{< bootstrap-table "table table-bordered table-striped table-responsive" >}} +| Ingress-NGINX Controller | NGINX Ingress Controller | NGINX Directive | +| ------------------------ | ------------------------ | --------------- | +| [_nginx.ingress.kubernetes.io/configuration-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#configuration-snippet) | [_nginx.org/location-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#snippets-and-custom-templates) | N/A | +| [_nginx.ingress.kubernetes.io/load-balance_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#custom-nginx-load-balancing) (1) | [_nginx.org/lb-method_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#backend-services-upstreams) | [_random two least_conn_](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#random) | +| [_nginx.ingress.kubernetes.io/proxy-buffering_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#proxy-buffering) | [_nginx.org/proxy-buffering_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_buffering_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) | +| [_nginx.ingress.kubernetes.io/proxy-buffers-number_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#proxy-buffers-number) | [_nginx.org/proxy-buffers_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_buffers_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) | +| [_nginx.ingress.kubernetes.io/proxy-buffer-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#proxy-buffer-size) | [_nginx.org/proxy-buffer-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_buffer_size_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) | +| [_nginx.ingress.kubernetes.io/proxy-connect-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#custom-timeouts) | [_nginx.org/proxy-connect-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_connect_timeout_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) | +| [_nginx.ingress.kubernetes.io/proxy-read-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#custom-timeouts) | [_nginx.org/proxy-read-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_read_timeout_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) | +| [_nginx.ingress.kubernetes.io/proxy-send-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#custom-timeouts) | [_nginx.org/proxy-send-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_send_timeout_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout) | +| [_nginx.ingress.kubernetes.io/rewrite-target_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#rewrite) | [_nginx.org/rewrites_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#request-uriheader-manipulation) | [_rewrite_](https://nginx.org/en/docs/http/ngx_http_rewrite_module.html#rewrite) | +| [_nginx.ingress.kubernetes.io/server-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#server-snippet)| [_nginx.org/server-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#snippets-and-custom-templates) | N/A | +| [_nginx.ingress.kubernetes.io/ssl-redirect_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#server-side-https-enforcement-through-redirect) | [_ingress.kubernetes.io/ssl-redirect_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#auth-and-ssltls) | N/A (2) | +{{% /bootstrap-table %}} + +1. Ingress-NGINX Controller implements some of its load balancing algorithms with Lua, which may not have an equivalent in NGINX Ingress Controller. +1. To redirect HTTP (80) traffic to HTTPS (443), NGINX Ingress Controller uses built-in NGINX `if` conditions while Ingress-NGINX Controller uses Lua. + +The following two snippets outline Ingress-NGINX Controller annotations that correspond to annotations for NGINX Ingress Controller with NGINX Plus. + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/affinity: "cookie" +nginx.ingress.kubernetes.io/session-cookie-name: "cookie_name" +nginx.ingress.kubernetes.io/session-cookie-expires: "seconds" +nginx.ingress.kubernetes.io/session-cookie-path: "/route" +``` + +**NGINX Ingress Controller (with NGINX Plus)** +```yaml +nginx.com/sticky-cookie-services: "serviceName=example-svc cookie_name expires=time path=/route" +``` + +{{< note >}} +NGINX Ingress Controller has additional annotations for features using NGINX Plus that have no Ingress-NGINX Controller equivalent, such as active health checks and authentication using JSON Web Tokens (JWTs). +{{< /note >}} + +### Global configuration with ConfigMaps + +This table maps the Ingress-NGINX Controller ConfigMap keys to NGINX Ingress Controller's equivalent ConfigMap keys. + + + +{{< bootstrap-table "table table-bordered table-striped table-responsive" >}} +| Ingress-NGINX Controller | NGINX Ingress Controller | +| ------------------------ | ------------------------ | +| [_disable-access-log_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#disable-access-log) | [_access-log-off_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_error-log-level_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#error-log-level) | [_error-log-level_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_hsts_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#hsts) | [_hsts_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_hsts-include-subdomains_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#hsts-include-subdomains) | [_hsts-include-subdomains_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_hsts-max-age_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#hsts-max-age) | [_hsts-max-age_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_http-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#http-snippet) | [_http-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#snippets-and-custom-templates) | +| [_keep-alive_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#keep-alive) | [_keepalive-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_keep-alive-requests_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#keep-alive-requests) | [_keepalive-requests_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_load-balance_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#load-balance) | [_lb-method_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#backend-services-upstreams) | +| [_location-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#location-snippet) | [_location-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#snippets-and-custom-templates) | +| [_log-format-escape-json_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#log-format-escape-json) | [_log-format-escaping: "json"_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_log-format-stream_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#log-format-stream) | [_stream-log-format_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_log-format-upstream_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#log-format-upstream) | [_log-format_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_main-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#main-snippet) | [_main-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#snippets-and-custom-templates) | +| [_max-worker-connections_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-connections) | [_worker-connections_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_max-worker-open-files_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-open-files) | [_worker-rlimit-nofile_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-body-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-body-size) | [_client-max-body-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-buffering_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-buffering) | [_proxy-buffering_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-buffers-number_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-buffers-number) | [_proxy-buffers: number size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-buffer-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-buffer-size) | [_proxy-buffers: number size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-connect-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-connect-timeout) | [_proxy-connect-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-read-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-read-timeout) | [_proxy-read-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-send-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-send-timeout) | [_proxy-send-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_server-name-hash-bucket-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#server-name-hash-bucket-size) | [_server-names-hash-bucket-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-headers-hash-max-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-headers-hash-max-size) | [_server-names-hash-max-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_server-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#server-snippet) | [_server-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#snippets-and-custom-templates) | +| [_server-tokens _](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#server-tokens) | [_server-tokens_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_ssl-ciphers_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#ssl-ciphers) | [_ssl-ciphers_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_ssl-dh-param_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#ssl-dh-param) | [_ssl-dhparam-file_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_ssl-protocols_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#ssl-protocols) | [_ssl-protocols_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_ssl-redirect_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#ssl-redirect) | [_ssl-redirect_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_upstream-keepalive-connections_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#upstream-keepalive-connections) | [_keepalive_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#backend-services-upstreams) | +| [_use-http2_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#use-http2) | [_http2_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#listeners) | +| [_use-proxy-protocol_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#use-proxy-protocol) | [_proxy-protocol_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#listeners) | +| [_variables-hash-bucket-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#variables-hash-bucket-size) | [_variables-hash-bucket-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_worker-cpu-affinity_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#worker-cpu-affinity) | [_worker-cpu-affinity_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_worker-processes_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#worker-processes) | [_worker-processes_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_worker-shutdown-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#worker-shutdown-timeout) | [_worker-shutdown-timeole_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +{{% /bootstrap-table %}} diff --git a/content/nic/installation/installing-nic/_index.md b/content/nic/installation/installing-nic/_index.md new file mode 100644 index 000000000..ad70f8f42 --- /dev/null +++ b/content/nic/installation/installing-nic/_index.md @@ -0,0 +1,5 @@ +--- +title: Install NGINX Ingress Controller +description: +weight: 100 +--- diff --git a/content/nic/installation/installing-nic/deploy-with-nap-using-helm.md b/content/nic/installation/installing-nic/deploy-with-nap-using-helm.md new file mode 100644 index 000000000..93cf807fd --- /dev/null +++ b/content/nic/installation/installing-nic/deploy-with-nap-using-helm.md @@ -0,0 +1,343 @@ +--- +title: Install NGINX Ingress Controller and NGINX App Protect WAF with Docker and Helm +toc: true +weight: 500 +type: how-to +product: NIC +--- + +This document describes how to build a local F5 NGINX App Protect WAF v5 Docker image with NGINX Plus Ingress +Controller, which can be used to compile WAF policies. + +This is accomplished with the following steps: + +- Prepare license secrets to enable a Kubernetes deployment +- Use a NGINX App Protect WAF Docker image to transform a policy JSON file into a compiled bundle +- Configure PersistentVolumes so the deployed NGINX App Protect WAF instance can access the compiled bundle +- Deploy NGINX Plus Ingress Controller with NGINX App Protect +- Test example services to validate that the WAF policies work + +--- + +## Prepare Secrets and credentials + +1. Download your NGINX Ingress Controller subscription's JSON Web Token, SSL Certificate, and Private Key from MyF5. + You can use the same JSON Web Token, Certificate, and Key as NGINX Plus in your MyF5 portal. +1. Rename the files to the following: + - `nginx-repo.crt` + - `nginx-repo.key` + - `nginx-repo.jwt` +1. Log in to the Docker registry using the contents of the JSON Web Token file: + ```shell + docker login private-registry.nginx.com --username=$(cat nginx-repo.jwt) --password=none + ``` + +--- + +## Compile WAF Policy from JSON to Bundle + +Pull the `waf-compiler` image with: + +```shell +docker pull private-registry.nginx.com/nap/waf-compiler:5.6.0 +``` + +Download the [provided WAF Policy JSON](https://raw.githubusercontent.com/nginx/kubernetes-ingress/main/tests/data/ap-waf-v5/wafv5.json): + +```shell +curl -L https://raw.githubusercontent.com/nginx/kubernetes-ingress/main/tests/data/ap-waf-v5/wafv5.json -o /tmp/wafv5.json +``` + +Use your pulled NAP Docker image (`private-registry.nginx.com/nap/waf-compiler:5.6.0`) to compile the policy bundle: + +```shell +# Using your newly created image +docker run --rm \ + -v /tmp:/tmp \ + private-registry.nginx.com/nap/waf-compiler:5.6.0 \ + -p /tmp/wafv5.json \ + -o /tmp/compiled_policy.tgz +``` + +Move the downloaded JSON and compiled policy to your workspace: + +```shell +mv /tmp/wafv5.json $(pwd)/wafv5.json +mv /tmp/compiled_policy.tgz $(pwd)/compiled_policy.tgz +``` + +After this command, your workspace should contain: + +``` +├── nginx-repo.crt +├── nginx-repo.key +├── nginx-repo.jwt +├── wafv5.json +└── compiled_policy.tgz +``` + +--- + +## Create the persistent volume and claim to store the policy bundle + +Save the following configuration data as `pvc.yaml` in the same directory. + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: task-pv-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/tmp/" + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: task-pv-claim +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +``` + +This sets up a 1Gi disk and attaches a claim to it that you will reference in the deployment chart. + +Create these with: + +```shell +kubectl apply -f pvc.yaml +``` + +Verify that the persistent volume and claim are created: + +```shell +# For the persistent volume +kubectl get pv + +# For the persistent volume claim +kubectl get pvc +``` + +## Deploy NGINX Plus NIC Controller with NAP Enabled using Helm + +Add the official NGINX Helm repository: + +```shell +helm repo add nginx-stable https://helm.nginx.com/stable +helm repo update +``` + +Create Kubernetes Docker and licensing secrets: +```shell +kubectl create secret \ + docker-registry regcred \ + --docker-server=private-registry.nginx.com \ + --docker-username=$(cat nginx-repo.jwt) \ + --docker-password=none + +kubectl create secret \ + generic license-token \ + --from-file=license.jwt=./nginx-repo.jwt \ + --type=nginx.com/license +``` + +Install the required CRDs for NGINX Ingress Controller: + +```shell +kubectl apply -f https://raw.githubusercontent.com/nginx/kubernetes-ingress/v5.0.0/deploy/crds.yaml +``` + +Using Helm, install NGINX Ingress Controller + +```shell +helm upgrade --install nic nginx-stable/nginx-ingress \ + --set controller.image.repository="private-registry.nginx.com/nginx-ic-nap-v5/nginx-plus-ingress" \ + --set controller.image.tag="5.0.0-alpine-fips" \ + --set controller.nginxplus=true \ + --set controller.appprotect.enable=true \ + --set controller.appprotect.v5=true \ + --set-json 'controller.appprotect.volumes=[ + {"name":"app-protect-bd-config","emptyDir":{}}, + {"name":"app-protect-config","emptyDir":{}}, + {"name":"app-protect-bundles","persistentVolumeClaim":{"claimName":"task-pv-claim"}} + ]' \ + --set controller.serviceAccount.imagePullSecretName=regcred \ + --set 'controller.volumeMounts[0].name=app-protect-bundles' \ + --set 'controller.volumeMounts[0].mountPath=/etc/app_protect/bundles/' + +``` + +Verify deployment success: + +```shell +kubectl get pods +``` + +--- + +## Copy the policy bundle into the running instance + +Get the name of the pod from the `kubectl get pods` command above. + +Copy the file into the `nginx-ingress` container within the pod: + +```shell +kubectl cp ./compiled_policy.tgz \ + :/etc/app_protect/bundles/compiled_policy.tgz \ + -c nginx-ingress +``` + +Replace `` with the actual name of the pod, for example: + +```shell +kubectl cp ./compiled_policy.tgz \ + nic-nginx-ingress-controller-9bd89589d-j925h:/etc/app_protect/bundles/compiled_policy.tgz \ + -c nginx-ingress +``` + +Confirm that the policy file is in the pod. The following command should list `compiled_policy.tgz`. + +```shell +kubectl exec --stdin --tty \ + -c nginx-ingress \ + \ + -- ls -la /etc/app_protect/bundles +``` + +## Confirm that the WAF policies work + +Save the following kubernetes config file as `webapp.yaml`: + +```yaml +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: webapp +spec: + host: webapp.example.com + policies: + - name: waf-policy + upstreams: + - name: webapp + service: webapp-svc + port: 80 + routes: + - path: / + action: + pass: webapp +--- +apiVersion: k8s.nginx.org/v1 +kind: Policy +metadata: + name: waf-policy +spec: + waf: + enable: true + apBundle: "compiled_policy.tgz" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: webapp +spec: + replicas: 1 + selector: + matchLabels: + app: webapp + template: + metadata: + labels: + app: webapp + spec: + containers: + - name: webapp + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: webapp-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: webapp +``` + +Create the services with +```shell +kubectl apply -f webapp.yaml +``` + +Confirm that the services have started with + +```shell +kubectl get pods +``` + +### Save the public IP and PORT in environment variables + +Get the public IP and port of your instance with the following command: + +```shell +kubectl get svc +``` + +Save them in the following environment variables: + +```shell +IC_IP=XXX.YYY.ZZZ.III +IC_HTTP_PORT= +``` + +### Validate that the WAF works + +Send a valid request to the deployed application: + +```shell +curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP http://webapp.example.com:$IC_HTTP_PORT/ +``` + +```text +Server address: 10.92.2.13:8080 +Server name: webapp-7b7dfbff54-dtxzt +Date: 18/Apr/2025:19:39:18 +0000 +URI: / +Request ID: 4f378a01fb8a36ae27e2c3059d264527 +``` + +And send one that should be rejected + +```shell +curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP "http://webapp.example.com:$IC_HTTP_PORT/