From 200b78db912bcdc2f294f5e64d8843fff8024215 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 17 Jan 2025 10:18:59 +0000 Subject: [PATCH 1/6] allow jwt_claim_ variables to be used in rate limiting --- .../rate-limit-jwt-claim/README.md | 79 +++++++++++++++++++ .../rate-limit-jwt-claim/rate-limit.yaml | 9 +++ .../rate-limit-jwt-claim/token.jwt | 1 + .../rate-limit-jwt-claim/virtual-server.yaml | 16 ++++ .../rate-limit-jwt-claim/webapp.yaml | 32 ++++++++ pkg/apis/configuration/validation/policy.go | 2 +- 6 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 examples/custom-resources/rate-limit-jwt-claim/README.md create mode 100644 examples/custom-resources/rate-limit-jwt-claim/rate-limit.yaml create mode 100644 examples/custom-resources/rate-limit-jwt-claim/token.jwt create mode 100644 examples/custom-resources/rate-limit-jwt-claim/virtual-server.yaml create mode 100644 examples/custom-resources/rate-limit-jwt-claim/webapp.yaml diff --git a/examples/custom-resources/rate-limit-jwt-claim/README.md b/examples/custom-resources/rate-limit-jwt-claim/README.md new file mode 100644 index 000000000..f2a6581fc --- /dev/null +++ b/examples/custom-resources/rate-limit-jwt-claim/README.md @@ -0,0 +1,79 @@ +# Rate Limit JWT claim + +In this example, we deploy a web application, configure load balancing for it via a VirtualServer, and apply a rate +limit policy using a JWT claim as the key to the rate limit. + +## Prerequisites + +1. Follow the [installation](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/) + instructions to deploy the Ingress Controller. +1. Save the public IP address of the Ingress Controller into a shell variable: + + ```console + IC_IP=XXX.YYY.ZZZ.III + ``` + +1. Save the HTTP port of the Ingress Controller into a shell variable: + + ```console + IC_HTTP_PORT= + ``` + +## Step 1 - Deploy a Web Application + +Create the application deployment and service: + +```console +kubectl apply -f webapp.yaml +``` + +## Step 2 - Deploy the Rate Limit Policy + +In this step, we create a policy with the name `rate-limit-jwt` that allows only 1 request per second coming from a +single IP address. + +Create the policy: + +```console +kubectl apply -f rate-limit.yaml +``` + +## Step 3 - Configure Load Balancing + +Create a VirtualServer resource for the web application: + +```console +kubectl apply -f virtual-server.yaml +``` + +Note that the VirtualServer references the policy `rate-limit-jwt` created in Step 2. + +## Step 4 - Test the Configuration + +Let's test the configuration. If you access the application at a rate that exceeds one request per second, NGINX will +start rejecting your requests: + +```console +curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP http://webapp.example.com:$IC_HTTP_PORT/ -H "Authorization: Bearer: `cat token.jwt`" +``` + +```text +Server address: 10.8.1.19:8080 +Server name: webapp-dc88fc766-zr7f8 +. . . +``` + +```console +curl --resolve webapp.example.com:$IC_HTTP_PORT:$IC_IP http://webapp.example.com:$IC_HTTP_PORT/ -H "Authorization: Bearer: `cat token.jwt`" +``` + +```text + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+ + +``` + +> Note: The command result is truncated for the clarity of the example. diff --git a/examples/custom-resources/rate-limit-jwt-claim/rate-limit.yaml b/examples/custom-resources/rate-limit-jwt-claim/rate-limit.yaml new file mode 100644 index 000000000..8b5ce903a --- /dev/null +++ b/examples/custom-resources/rate-limit-jwt-claim/rate-limit.yaml @@ -0,0 +1,9 @@ +apiVersion: k8s.nginx.org/v1 +kind: Policy +metadata: + name: rate-limit-jwt +spec: + rateLimit: + rate: 1r/s + key: ${jwt_claim_sub} + zoneSize: 10M diff --git a/examples/custom-resources/rate-limit-jwt-claim/token.jwt b/examples/custom-resources/rate-limit-jwt-claim/token.jwt new file mode 100644 index 000000000..eacb21aa8 --- /dev/null +++ b/examples/custom-resources/rate-limit-jwt-claim/token.jwt @@ -0,0 +1 @@ +eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiIsImtpZCI6IjAwMDEifQ.eyJuYW1lIjoiUXVvdGF0aW9uIFN5c3RlbSIsInN1YiI6InF1b3RlcyIsImlzcyI6Ik15IEFQSSBHYXRld2F5In0.ggVOHYnVFB8GVPE-VOIo3jD71gTkLffAY0hQOGXPL2I diff --git a/examples/custom-resources/rate-limit-jwt-claim/virtual-server.yaml b/examples/custom-resources/rate-limit-jwt-claim/virtual-server.yaml new file mode 100644 index 000000000..ec59e2423 --- /dev/null +++ b/examples/custom-resources/rate-limit-jwt-claim/virtual-server.yaml @@ -0,0 +1,16 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: webapp +spec: + host: webapp.example.com + policies: + - name: rate-limit-jwt + upstreams: + - name: webapp + service: webapp-svc + port: 80 + routes: + - path: / + action: + pass: webapp diff --git a/examples/custom-resources/rate-limit-jwt-claim/webapp.yaml b/examples/custom-resources/rate-limit-jwt-claim/webapp.yaml new file mode 100644 index 000000000..31fde92a6 --- /dev/null +++ b/examples/custom-resources/rate-limit-jwt-claim/webapp.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: webapp +spec: + replicas: 1 + selector: + matchLabels: + app: webapp + template: + metadata: + labels: + app: webapp + spec: + containers: + - name: webapp + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: webapp-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: webapp diff --git a/pkg/apis/configuration/validation/policy.go b/pkg/apis/configuration/validation/policy.go index 9c7a72ad3..98d8626d0 100644 --- a/pkg/apis/configuration/validation/policy.go +++ b/pkg/apis/configuration/validation/policy.go @@ -569,7 +569,7 @@ func validateRateLimitZoneSize(zoneSize string, fieldPath *field.Path) field.Err return allErrs } -var rateLimitKeySpecialVariables = []string{"arg_", "http_", "cookie_"} +var rateLimitKeySpecialVariables = []string{"arg_", "http_", "cookie_", "jwt_claim_"} // rateLimitKeyVariables includes NGINX variables allowed to be used in a rateLimit policy key. var rateLimitKeyVariables = map[string]bool{ From 0555f654a75c4c084a35eb74eb34fda82dfd0660 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 21 Jan 2025 12:31:18 +0000 Subject: [PATCH 2/6] update rate limit policy documentation --- site/content/configuration/policy-resource.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/configuration/policy-resource.md b/site/content/configuration/policy-resource.md index 4c764194b..df6ea616a 100644 --- a/site/content/configuration/policy-resource.md +++ b/site/content/configuration/policy-resource.md @@ -121,7 +121,7 @@ The feature is implemented using the NGINX [ngx_http_limit_req_module](https://n |Field | Description | Type | Required | | ---| ---| ---| --- | |``rate`` | The rate of requests permitted. The rate is specified in requests per second (r/s) or requests per minute (r/m). | ``string`` | Yes | -|``key`` | The key to which the rate limit is applied. Can contain text, variables, or a combination of them. Variables must be surrounded by ``${}``. For example: ``${binary_remote_addr}``. Accepted variables are ``$binary_remote_addr``, ``$request_uri``, ``$url``, ``$http_``, ``$args``, ``$arg_``, ``$cookie_``. | ``string`` | Yes | +|``key`` | The key to which the rate limit is applied. Can contain text, variables, or a combination of them. Variables must be surrounded by ``${}``. For example: ``${binary_remote_addr}``. Accepted variables are ``$binary_remote_addr``, ``$request_uri``, ``$url``, ``$http_``, ``$args``, ``$arg_``, ``$cookie_``, ``$jwt_claim_``. | ``string`` | Yes | |``zoneSize`` | Size of the shared memory zone. Only positive values are allowed. Allowed suffixes are ``k`` or ``m``, if none are present ``k`` is assumed. | ``string`` | Yes | |``delay`` | The delay parameter specifies a limit at which excessive requests become delayed. If not set all excessive requests are delayed. | ``int`` | No | |``noDelay`` | Disables the delaying of excessive requests while requests are being limited. Overrides ``delay`` if both are set. | ``bool`` | No | From fad03beaed916e23f0b02700b7ea19efc169c686 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 12:17:10 +0000 Subject: [PATCH 3/6] chore(deps): bump the actions group with 2 updates (#7172) Bumps the actions group with 2 updates: [reviewdog/action-actionlint](https://github.com/reviewdog/action-actionlint) and [DavidAnson/markdownlint-cli2-action](https://github.com/davidanson/markdownlint-cli2-action). Updates `reviewdog/action-actionlint` from 1.63.0 to 1.64.1 - [Release notes](https://github.com/reviewdog/action-actionlint/releases) - [Commits](https://github.com/reviewdog/action-actionlint/compare/f3dcc52bc6039e5d736486952379dce3e869e8a2...abd537417cf4991e1ba8e21a67b1119f4f53b8e0) Updates `DavidAnson/markdownlint-cli2-action` from 19.0.0 to 19.1.0 - [Release notes](https://github.com/davidanson/markdownlint-cli2-action/releases) - [Commits](https://github.com/davidanson/markdownlint-cli2-action/compare/a23dae216ce3fee4db69da41fed90d2a4af801cf...05f32210e84442804257b2a6f20b273450ec8265) --- updated-dependencies: - dependency-name: reviewdog/action-actionlint dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: DavidAnson/markdownlint-cli2-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Venktesh Shivam Patel --- .github/workflows/lint-format.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-format.yml b/.github/workflows/lint-format.yml index cd4efc303..f4d2a1d29 100644 --- a/.github/workflows/lint-format.yml +++ b/.github/workflows/lint-format.yml @@ -63,7 +63,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: reviewdog/action-actionlint@f3dcc52bc6039e5d736486952379dce3e869e8a2 # v1.63.0 + - uses: reviewdog/action-actionlint@abd537417cf4991e1ba8e21a67b1119f4f53b8e0 # v1.64.1 with: actionlint_flags: -shellcheck "" @@ -84,7 +84,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: DavidAnson/markdownlint-cli2-action@a23dae216ce3fee4db69da41fed90d2a4af801cf # v19.0.0 + - uses: DavidAnson/markdownlint-cli2-action@05f32210e84442804257b2a6f20b273450ec8265 # v19.1.0 with: config: .markdownlint-cli2.yaml globs: "**/*.md" From 90a6427b1bc9f0d90eea3bb598f1c46e4268af62 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 21 Jan 2025 15:38:44 +0000 Subject: [PATCH 4/6] add python tests for rate limit with jwt claim --- .../policies/rate-limit-jwt-claim-sub.yaml | 9 +++ .../spec/virtual-server-jwt-claim-sub.yaml | 22 ++++++++ tests/suite/test_rl_policies.py | 55 +++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 tests/data/rate-limit/policies/rate-limit-jwt-claim-sub.yaml create mode 100644 tests/data/rate-limit/spec/virtual-server-jwt-claim-sub.yaml diff --git a/tests/data/rate-limit/policies/rate-limit-jwt-claim-sub.yaml b/tests/data/rate-limit/policies/rate-limit-jwt-claim-sub.yaml new file mode 100644 index 000000000..5038fd07a --- /dev/null +++ b/tests/data/rate-limit/policies/rate-limit-jwt-claim-sub.yaml @@ -0,0 +1,9 @@ +apiVersion: k8s.nginx.org/v1 +kind: Policy +metadata: + name: rate-limit-jwt-claim-sub +spec: + rateLimit: + rate: 1r/s + key: ${jwt_claim_sub} + zoneSize: 10M diff --git a/tests/data/rate-limit/spec/virtual-server-jwt-claim-sub.yaml b/tests/data/rate-limit/spec/virtual-server-jwt-claim-sub.yaml new file mode 100644 index 000000000..b1bc7c8d1 --- /dev/null +++ b/tests/data/rate-limit/spec/virtual-server-jwt-claim-sub.yaml @@ -0,0 +1,22 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: virtual-server +spec: + host: virtual-server.example.com + policies: + - name: rate-limit-jwt-claim-sub + upstreams: + - name: backend2 + service: backend2-svc + port: 80 + - name: backend1 + service: backend1-svc + port: 80 + routes: + - path: "/backend1" + action: + pass: backend1 + - path: "/backend2" + action: + pass: backend2 diff --git a/tests/suite/test_rl_policies.py b/tests/suite/test_rl_policies.py index bc167d7ff..c7f8b3d8b 100644 --- a/tests/suite/test_rl_policies.py +++ b/tests/suite/test_rl_policies.py @@ -24,6 +24,9 @@ rl_vs_override_spec = f"{TEST_DATA}/rate-limit/spec/virtual-server-override.yaml" rl_vs_override_route = f"{TEST_DATA}/rate-limit/route-subroute/virtual-server-override-route.yaml" rl_vs_override_spec_route = f"{TEST_DATA}/rate-limit/route-subroute/virtual-server-override-spec-route.yaml" +rl_vs_jwt_claim_sub = f"{TEST_DATA}/rate-limit/spec/virtual-server-jwt-claim-sub.yaml" +rl_pol_jwt_claim_sub = f"{TEST_DATA}/rate-limit/policies/rate-limit-jwt-claim-sub.yaml" +token = f"{TEST_DATA}/jwt-policy/token.jwt" @pytest.mark.policies @@ -357,3 +360,55 @@ def test_rl_policy_scaled( and policy_info["status"]["reason"] == "AddedOrUpdated" and policy_info["status"]["state"] == "Valid" ) + + @pytest.mark.skip_for_nginx_oss + @pytest.mark.parametrize("src", [rl_vs_jwt_claim_sub]) + def test_rl_policy_jwt_claim_sub( + self, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + virtual_server_setup, + test_namespace, + src, + ): + """ + Test if rate-limiting policy is working with 1 rps using $jwt_claim_sub as the rate limit key + """ + print(f"Create rl policy") + pol_name = create_policy_from_yaml(kube_apis.custom_objects, rl_pol_jwt_claim_sub, test_namespace) + print(f"Patch vs with policy: {src}") + patch_virtual_server_from_yaml( + kube_apis.custom_objects, + virtual_server_setup.vs_name, + src, + virtual_server_setup.namespace, + ) + wait_before_test(10) + + policy_info = read_custom_resource(kube_apis.custom_objects, test_namespace, "policies", pol_name) + occur = [] + t_end = time.perf_counter() + 1 + resp = requests.get( + virtual_server_setup.backend_1_url, + headers={"host": virtual_server_setup.vs_host, "Authorization": f"Bearer {token}"}, + ) + # wait_before_test(120) + print(resp.status_code) + assert resp.status_code == 200 + while time.perf_counter() < t_end: + resp = requests.get( + virtual_server_setup.backend_1_url, + headers={"host": virtual_server_setup.vs_host, "Authorization": f"Bearer {token}"}, + ) + print(resp.status_code) + occur.append(resp.status_code) + wait_before_test(300) + delete_policy(kube_apis.custom_objects, pol_name, test_namespace) + self.restore_default_vs(kube_apis, virtual_server_setup) + assert ( + policy_info["status"] + and policy_info["status"]["reason"] == "AddedOrUpdated" + and policy_info["status"]["state"] == "Valid" + ) + assert occur.count(200) <= 1 From 42393e2f5e57f0db18b0eea61b88894ecf922b37 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Wed, 22 Jan 2025 12:30:53 +0000 Subject: [PATCH 5/6] remove debug --- tests/suite/test_rl_policies.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/suite/test_rl_policies.py b/tests/suite/test_rl_policies.py index c7f8b3d8b..2d8185ea5 100644 --- a/tests/suite/test_rl_policies.py +++ b/tests/suite/test_rl_policies.py @@ -384,7 +384,7 @@ def test_rl_policy_jwt_claim_sub( src, virtual_server_setup.namespace, ) - wait_before_test(10) + wait_before_test() policy_info = read_custom_resource(kube_apis.custom_objects, test_namespace, "policies", pol_name) occur = [] @@ -393,17 +393,15 @@ def test_rl_policy_jwt_claim_sub( virtual_server_setup.backend_1_url, headers={"host": virtual_server_setup.vs_host, "Authorization": f"Bearer {token}"}, ) - # wait_before_test(120) print(resp.status_code) + wait_before_test() assert resp.status_code == 200 while time.perf_counter() < t_end: resp = requests.get( virtual_server_setup.backend_1_url, headers={"host": virtual_server_setup.vs_host, "Authorization": f"Bearer {token}"}, ) - print(resp.status_code) occur.append(resp.status_code) - wait_before_test(300) delete_policy(kube_apis.custom_objects, pol_name, test_namespace) self.restore_default_vs(kube_apis, virtual_server_setup) assert ( From 5416c6de31e6e5ae74300af33eaafc62ad4f4635 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Wed, 22 Jan 2025 16:43:28 +0000 Subject: [PATCH 6/6] Update readme --- .../rate-limit-jwt-claim/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/examples/custom-resources/rate-limit-jwt-claim/README.md b/examples/custom-resources/rate-limit-jwt-claim/README.md index f2a6581fc..ce48a1adf 100644 --- a/examples/custom-resources/rate-limit-jwt-claim/README.md +++ b/examples/custom-resources/rate-limit-jwt-claim/README.md @@ -50,7 +50,19 @@ Note that the VirtualServer references the policy `rate-limit-jwt` created in St ## Step 4 - Test the Configuration -Let's test the configuration. If you access the application at a rate that exceeds one request per second, NGINX will +The JWT payload used in this testing looks like: + +```json +{ + "name": "Quotation System", + "sub": "quotes", + "iss": "My API Gateway" +} +``` + +In this test we are relying on the NGINX Plus `ngx_http_auth_jwt_module` to extract the `sub` claim from the JWT payload into the `$jwt_claim_sub` variable and use this as the rate limiting `key`. + +Let's test the configuration. If you access the application at a rate that exceeds one request per second, NGINX will start rejecting your requests: ```console