diff --git a/.nvmrc b/.nvmrc index 2bd5a0a98a..a45fd52cc5 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -22 +24 diff --git a/Makefile b/Makefile index cc408afc54..a49147a7d8 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ GEN_CRD_API_REFERENCE_DOCS_VERSION = v0.3.0 # renovate: datasource=go depName=sigs.k8s.io/controller-tools CONTROLLER_TOOLS_VERSION = v0.19.0 # renovate: datasource=docker depName=node -NODE_VERSION = 22 +NODE_VERSION = 24 # renovate: datasource=docker depName=quay.io/helmpack/chart-testing CHART_TESTING_VERSION = v3.13.0 # renovate: datasource=github-tags depName=dadav/helm-schema diff --git a/build/Dockerfile.nginx b/build/Dockerfile.nginx index 83c7e066ae..d326c08984 100644 --- a/build/Dockerfile.nginx +++ b/build/Dockerfile.nginx @@ -23,7 +23,7 @@ RUN apk add --no-cache bash \ && ln -sf /dev/stderr /var/log/nginx/error.log COPY build/entrypoint.sh /agent/entrypoint.sh -COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js +COPY ${NJS_DIR}/ /usr/lib/nginx/modules/njs/ COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf diff --git a/build/Dockerfile.nginxplus b/build/Dockerfile.nginxplus index a33adefb0f..3828ce391f 100644 --- a/build/Dockerfile.nginxplus +++ b/build/Dockerfile.nginxplus @@ -29,7 +29,7 @@ RUN apk add --no-cache bash \ && ln -sf /dev/stderr /var/log/nginx/error.log COPY build/entrypoint.sh /agent/entrypoint.sh -COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js +COPY ${NJS_DIR}/ /usr/lib/nginx/modules/njs/ COPY ${NGINX_CONF_DIR}/nginx-plus.conf /etc/nginx/nginx.conf COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf diff --git a/internal/controller/nginx/conf/nginx-plus.conf b/internal/controller/nginx/conf/nginx-plus.conf index f2b0ec0dc8..50ba9f970c 100644 --- a/internal/controller/nginx/conf/nginx-plus.conf +++ b/internal/controller/nginx/conf/nginx-plus.conf @@ -13,6 +13,7 @@ http { include /etc/nginx/conf.d/*.conf; include /etc/nginx/mime.types; js_import /usr/lib/nginx/modules/njs/httpmatches.js; + js_import /usr/lib/nginx/modules/njs/epp.js; default_type application/octet-stream; diff --git a/internal/controller/nginx/conf/nginx.conf b/internal/controller/nginx/conf/nginx.conf index 791994fdf8..6c4f6be8d9 100644 --- a/internal/controller/nginx/conf/nginx.conf +++ b/internal/controller/nginx/conf/nginx.conf @@ -13,6 +13,7 @@ http { include /etc/nginx/conf.d/*.conf; include /etc/nginx/mime.types; js_import /usr/lib/nginx/modules/njs/httpmatches.js; + js_import /usr/lib/nginx/modules/njs/epp.js; default_type application/octet-stream; diff --git a/internal/controller/nginx/modules/README.md b/internal/controller/nginx/modules/README.md index 9c7c805276..3313ea6604 100644 --- a/internal/controller/nginx/modules/README.md +++ b/internal/controller/nginx/modules/README.md @@ -22,6 +22,7 @@ dependencies. - [httpmatches](./src/httpmatches.js): a location handler for HTTP requests. It redirects requests to an internal location block based on the request's headers, arguments, and method. +- [epp](./src/epp.js): handles communication with the EndpointPicker (EPP) component. This is for acquiring a specific AI endpoint to route client traffic to when using the Gateway API Inference Extension. ### Helpful Resources for Module Development diff --git a/internal/controller/nginx/modules/src/epp.js b/internal/controller/nginx/modules/src/epp.js new file mode 100644 index 0000000000..8efcd70ece --- /dev/null +++ b/internal/controller/nginx/modules/src/epp.js @@ -0,0 +1,29 @@ +// This file contains the methods to get an AI workload endpoint from the EndpointPicker (EPP). + +// TODO (sberman): this module will need to be enhanced to include the following: +// - function that sends the subrequest to the Go middleware application (to get the endpoint from EPP) +// - if a user has specified an Exact matching condition for a model name, extract the model name from +// the request body, and if it matches that condition, set the proper value in the X-Gateway-Model-Name header +// (based on if we do a redirect or traffic split (see design doc)) in the subrequest. If the client request +// already has this header set, then I don't think we need to extract the model from the body, just pass +// through the existing header. +// I believe we have to use js_content to call the NJS functionality. Because this takes over +// the request, we will likely have to finish the NJS functionality with an internalRedirect to an internal +// location that proxy_passes to the chosen endpoint. + +// extractModel extracts the model name from the request body. +function extractModel(r) { + try { + var body = JSON.parse(r.requestText); + if (body && body.model !== undefined) { + return String(body.model); + } + } catch (e) { + r.error(`error parsing request body for model name: ${e.message}`); + return ''; + } + r.error('request body does not contain model parameter'); + return ''; +} + +export default { extractModel }; diff --git a/internal/controller/nginx/modules/test/epp.test.js b/internal/controller/nginx/modules/test/epp.test.js new file mode 100644 index 0000000000..6994423e7a --- /dev/null +++ b/internal/controller/nginx/modules/test/epp.test.js @@ -0,0 +1,52 @@ +import { default as epp } from '../src/epp.js'; +import { expect, describe, it } from 'vitest'; + +function makeRequest(body) { + let r = { + // Test mocks + error(msg) { + r.variables.error = msg; + }, + requestText: body, + variables: {}, + }; + + return r; +} + +describe('extractModel', () => { + const tests = [ + { + name: 'returns the model value', + body: '{"model":"gpt-4"}', + model: 'gpt-4', + error: undefined, + }, + { + name: 'returns empty string if model is missing', + body: '{"foo":1}', + model: '', + error: 'request body does not contain model parameter', + }, + { + name: 'returns empty string for invalid JSON', + body: 'not-json', + model: '', + error: `error parsing request body for model name: Unexpected token 'o', "not-json" is not valid JSON`, + }, + { + name: 'empty request body', + body: '', + model: '', + error: 'error parsing request body for model name: Unexpected end of JSON input', + }, + ]; + + tests.forEach((test) => { + it(test.name, () => { + let r = makeRequest(test.body); + expect(epp.extractModel(r)).to.equal(test.model); + expect(r.variables.error).to.equal(test.error); + }); + }); +});