From c275ac846aa09b025c6c476a5cdc4ff549c41f68 Mon Sep 17 00:00:00 2001 From: William May Date: Tue, 9 Sep 2025 16:38:40 +0100 Subject: [PATCH 01/11] FLS-1452 - Rename RegisterFormPublishApi to RegisterFormsApi The previous name felt overly-specific, given that the API endpoints defined in this file also include ones that are foundational to the application, including for getting form pages and submitting form data. Plus, we're about to remove the publish functionality. --- runner/src/server/plugins/engine/MainPlugin.ts | 4 ++-- .../api/{RegisterFormPublishApi.ts => RegisterFormsApi.ts} | 2 +- runner/src/server/plugins/engine/api/index.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename runner/src/server/plugins/engine/api/{RegisterFormPublishApi.ts => RegisterFormsApi.ts} (99%) diff --git a/runner/src/server/plugins/engine/MainPlugin.ts b/runner/src/server/plugins/engine/MainPlugin.ts index 3ec372e8..1c616238 100644 --- a/runner/src/server/plugins/engine/MainPlugin.ts +++ b/runner/src/server/plugins/engine/MainPlugin.ts @@ -1,6 +1,6 @@ import {Options} from "./types/PluginOptions"; import {HapiServer} from "../../types"; -import {RegisterFormPublishApi} from "./api"; +import {RegisterFormsApi} from "./api"; const LOGGER_DATA = { @@ -32,6 +32,6 @@ export const plugin = { ...LOGGER_DATA, message: `[FORM-CACHE] number of forms loaded into cache ok[${countOk}] error[${countError}]` }) - new RegisterFormPublishApi().register(server, options); + new RegisterFormsApi().register(server, options); } }; diff --git a/runner/src/server/plugins/engine/api/RegisterFormPublishApi.ts b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts similarity index 99% rename from runner/src/server/plugins/engine/api/RegisterFormPublishApi.ts rename to runner/src/server/plugins/engine/api/RegisterFormsApi.ts index 75033c32..a42ea552 100644 --- a/runner/src/server/plugins/engine/api/RegisterFormPublishApi.ts +++ b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts @@ -12,7 +12,7 @@ import {PluginSpecificConfiguration} from "@hapi/hapi"; import {jwtAuthStrategyName} from "../Auth"; import {config} from "../../utils/AdapterConfigurationSchema"; -export class RegisterFormPublishApi implements RegisterApi { +export class RegisterFormsApi implements RegisterApi { /** * The following publish endpoints (/publish, /published/{id}, /published) diff --git a/runner/src/server/plugins/engine/api/index.ts b/runner/src/server/plugins/engine/api/index.ts index 137726a5..562ed52c 100644 --- a/runner/src/server/plugins/engine/api/index.ts +++ b/runner/src/server/plugins/engine/api/index.ts @@ -2,4 +2,4 @@ export {RegisterSessionApi} from "./RegisterSessionApi" export {RegisterApi} from "./RegisterApi" export {RegisterApplicationStatusApi} from "./RegisterApplicationStatusApi" export {RegisterPublicApi} from "./RegisterPublicApi" -export {RegisterFormPublishApi} from "./RegisterFormPublishApi" +export {RegisterFormsApi} from "./RegisterFormsApi" From 7b49763d408ade4b4ee2802c3315836eae9dbc78 Mon Sep 17 00:00:00 2001 From: William May Date: Tue, 9 Sep 2025 17:58:48 +0100 Subject: [PATCH 02/11] FLS-1452 - Remove environmental preview mode 'Preview mode' is an environment variable that is currently set on an environment-by-environment basis. It controls whether we allow passage through the application's 'publishing' endpoints. These publishing endpoints, which support FAB and Form Designer to offer form preview even where forms are not loaded into the Runner cache at app startup, are being removed as part of Live Services' forms re-architecture work. In future, instead of the Runner supporting publishing of arbitrary forms into its cache, all forms will exist in a shared location in Pre-Award, and there will be no practical use case for rendering an arbitrary form. Please note that removing this environment variable does not itself stop forms being previewed, or the preview banner from being rendered when a form is previewed. This is because the preview banner rendering is dependent on the prefix 'preview' being present at the beginning of the form session identifier retrieved from the URL. --- .run/RUNNER.run.xml | 1 - copilot/fsd-form-runner-adapter/manifest.yml | 4 -- docker-compose.e2e.yml | 2 - docker-compose.yml | 1 - .../config/custom-environment-variables.json | 1 - runner/config/default.js | 1 - runner/config/development.json | 1 - runner/config/test.json | 1 - .../src/server/plugins/engine/MainPlugin.ts | 2 +- .../plugins/engine/api/RegisterFormsApi.ts | 38 +------------------ 10 files changed, 3 insertions(+), 49 deletions(-) diff --git a/.run/RUNNER.run.xml b/.run/RUNNER.run.xml index 219ebb11..c2135770 100644 --- a/.run/RUNNER.run.xml +++ b/.run/RUNNER.run.xml @@ -29,7 +29,6 @@ - diff --git a/copilot/fsd-form-runner-adapter/manifest.yml b/copilot/fsd-form-runner-adapter/manifest.yml index d9ed5e46..f551af55 100644 --- a/copilot/fsd-form-runner-adapter/manifest.yml +++ b/copilot/fsd-form-runner-adapter/manifest.yml @@ -84,7 +84,6 @@ environments: dev: variables: JWT_AUTH_ENABLED: false - PREVIEW_MODE: true count: spot: 2 sidecars: @@ -107,7 +106,6 @@ environments: test: variables: JWT_AUTH_ENABLED: false - PREVIEW_MODE: true count: spot: 2 sidecars: @@ -130,7 +128,6 @@ environments: uat: variables: JWT_AUTH_ENABLED: false - PREVIEW_MODE: true count: range: 2-4 cooldown: @@ -174,7 +171,6 @@ environments: SERVICE_START_PAGE: "https://apply.access-funding.communities.gov.uk/account" ELIGIBILITY_RESULT_URL: "https://apply.access-funding.communities.gov.uk/eligibility-result" SENTRY_TRACES_SAMPLE_RATE: 0.1 - PREVIEW_MODE: false count: range: 2-4 cooldown: diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index 0c8e5de4..897403d0 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -25,7 +25,6 @@ services: - "3009:3009" environment: - CHOKIDAR_USEPOLLING=true - - PREVIEW_MODE=true - LAST_COMMIT - LAST_TAG - ACCESSIBILITY_STATEMENT_URL=http://localhost:3008/accessibility_statement @@ -43,7 +42,6 @@ services: - SERVICE_START_PAGE=http://localhost:3008/account - SINGLE_REDIS=true - FORM_RUNNER_ADAPTER_REDIS_INSTANCE_URI=redis://redis-data:6379 - - PREVIEW_MODE=true - ENABLE_VIRUS_SCAN=true command: yarn runner start:test logging: diff --git a/docker-compose.yml b/docker-compose.yml index ef1e94de..2b5bccc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,6 @@ services: - "3009:3009" environment: - CHOKIDAR_USEPOLLING=true - - PREVIEW_MODE=true - LAST_COMMIT - LAST_TAG - JWT_AUTH_ENABLED=false diff --git a/runner/config/custom-environment-variables.json b/runner/config/custom-environment-variables.json index 041d478b..4952c951 100644 --- a/runner/config/custom-environment-variables.json +++ b/runner/config/custom-environment-variables.json @@ -45,7 +45,6 @@ "authClientAuthUrl": "AUTH_CLIENT_AUTH_URL", "authClientTokenUrl": "AUTH_CLIENT_TOKEN_URL", "authClientProfileUrl": "AUTH_CLIENT_PROFILE_URL", - "previewMode": "PREVIEW_MODE", "enforceCsrf": "ENFORCE_CSRF", "savePerPage": "SAVE_PER_PAGE", "awsBucketName": "AWS_BUCKET_NAME", diff --git a/runner/config/default.js b/runner/config/default.js index 4049a989..f43bb6aa 100644 --- a/runner/config/default.js +++ b/runner/config/default.js @@ -21,7 +21,6 @@ module.exports = { */ port: 3009, env: "development", - previewMode: false, enforceCsrf: true, singleRedis: false, isE2EModeEnabled: false, diff --git a/runner/config/development.json b/runner/config/development.json index be126c86..9c666686 100644 --- a/runner/config/development.json +++ b/runner/config/development.json @@ -1,6 +1,5 @@ { "isTest": true, - "previewMode": true, "enforceCsrf": false, "env": "development" } diff --git a/runner/config/test.json b/runner/config/test.json index 0d143677..3613637a 100644 --- a/runner/config/test.json +++ b/runner/config/test.json @@ -1,7 +1,6 @@ { "safelist": ["webho.ok"], "isTest": true, - "previewMode": true, "enforceCsrf": false, "initialisedSessionKey": "predictable-key", "env": "test" diff --git a/runner/src/server/plugins/engine/MainPlugin.ts b/runner/src/server/plugins/engine/MainPlugin.ts index 1c616238..1378fc07 100644 --- a/runner/src/server/plugins/engine/MainPlugin.ts +++ b/runner/src/server/plugins/engine/MainPlugin.ts @@ -32,6 +32,6 @@ export const plugin = { ...LOGGER_DATA, message: `[FORM-CACHE] number of forms loaded into cache ok[${countOk}] error[${countError}]` }) - new RegisterFormsApi().register(server, options); + new RegisterFormsApi().register(server); } }; diff --git a/runner/src/server/plugins/engine/api/RegisterFormsApi.ts b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts index a42ea552..22bf7bcc 100644 --- a/runner/src/server/plugins/engine/api/RegisterFormsApi.ts +++ b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts @@ -1,6 +1,5 @@ import {RegisterApi} from "./RegisterApi"; import {HapiRequest, HapiResponseToolkit, HapiServer} from "../../../types"; -import {Options} from "../types/PluginOptions"; import {FormPayload} from "../../../../../../digital-form-builder/runner/src/server/plugins/engine/types"; // @ts-ignore import Boom from "boom"; @@ -22,11 +21,7 @@ export class RegisterFormsApi implements RegisterApi { * for its own purposes so if you're changing these endpoints you likely need to go and amend * the designer too! */ - register(server: HapiServer, options: Options) { - const {previewMode} = options; - const disabledRouteDetailString = - "A request was made however previewing is disabled. See environment variable details in runner/README.md if this error is not expected."; - + register(server: HapiServer) { server.route({ method: "post", path: "/publish", @@ -35,17 +30,8 @@ export class RegisterFormsApi implements RegisterApi { }, handler: async (request: HapiRequest, h: HapiResponseToolkit) => { const {adapterCacheService} = request.services([]); - // @ts-ignore - if (!previewMode || previewMode==="false") { - request.logger.error( - [`POST /publish`, "previewModeError"], - disabledRouteDetailString - ); - throw Boom.forbidden("Publishing is disabled"); - } const payload = request.payload as FormPayload; const {id, configuration} = payload; - const parsedConfiguration = typeof configuration === "string" ? JSON.parse(configuration) @@ -67,14 +53,6 @@ export class RegisterFormsApi implements RegisterApi { }, handler: async (request: HapiRequest, h: HapiResponseToolkit) => { const {id} = request.params; - // @ts-ignore - if (!previewMode || previewMode==="false") { - request.logger.error( - [`GET /published/${id}`, "previewModeError"], - disabledRouteDetailString - ); - throw Boom.unauthorized("publishing is disabled"); - } const {adapterCacheService} = request.services([]); const form = await adapterCacheService.getFormAdapterModel(id, request); if (!form) { @@ -93,14 +71,6 @@ export class RegisterFormsApi implements RegisterApi { }, handler: async (request: HapiRequest, h: HapiResponseToolkit) => { const {adapterCacheService} = request.services([]); - // @ts-ignore - if (!previewMode || previewMode==="false") { - request.logger.error( - [`GET /published`, "previewModeError"], - disabledRouteDetailString - ); - throw Boom.unauthorized("publishing is disabled."); - } return h .response(JSON.stringify(await adapterCacheService.getFormConfigurations(request))) .code(200); @@ -163,7 +133,6 @@ export class RegisterFormsApi implements RegisterApi { /** * Middleware to check if the user session is still valid. * - * Changes behaviour only when previewMode is FALSE, meaning PRODUCTION * If the session is dropped, it will throw a client timeout error */ const checkUserSession = async ( @@ -175,10 +144,7 @@ export class RegisterFormsApi implements RegisterApi { // @ts-ignore const state = await adapterCacheService.getState(request); - // @ts-ignore isNotPreview is always false on production - const isNotPreview = !previewMode || previewMode==="false" - - if (isNotPreview && !state.callback) { + if (config.copilotEnv == "prod" && !state.callback) { // if you are here the session likely dropped request.logger.error(["checkUserSession"], `Session expired ${request.yar.id}`); From e9c08299178763e8983bb89be26c4e68bc3a5922 Mon Sep 17 00:00:00 2001 From: William May Date: Tue, 9 Sep 2025 18:14:45 +0100 Subject: [PATCH 03/11] FLS-1452 - Remove publishing endpoints Publishing endpoints in Form Runner are used as Form Designer's primary persistence mechanism - each change made to a form in Form Designer will trigger a POST request to the /publish endpoint, updating the form configuration in the Runner's cache. They are also used as a means to allow FAB and Form Designer to offer form previews for forms that are not loaded into the Runner cache on app startup. Thus they are integral to the current functioning of the application, and this commit can very much be considered a potentially breaking change, so it's important that FAB and Form Designer are redirected at Pre-Award before this goes live. Why are we removing them then? They are components of a flawed system that is being revamped by Live Services as part of our forms re-architecture work. In the future, there will be no need for arbitrary forms to be published into the cache, in order to support persistence or preview. Forms will be stored in a new location - the Pre-Award database - and Form Designer will POST updated form configuration data to a Pre-Award API. And regarding form preview, for a form to even be shown in Form Designer or FAB it must exist in the Pre-Award database, and so rendering it from Form Designer or FAB will be a simple case of requesting the form directly from the Runner, which will get it from the Pre-Award API, with no precursor publish call necessary. --- .../plugins/engine/api/RegisterFormsApi.ts | 64 ------------------- 1 file changed, 64 deletions(-) diff --git a/runner/src/server/plugins/engine/api/RegisterFormsApi.ts b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts index 22bf7bcc..4432d21f 100644 --- a/runner/src/server/plugins/engine/api/RegisterFormsApi.ts +++ b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts @@ -12,71 +12,7 @@ import {jwtAuthStrategyName} from "../Auth"; import {config} from "../../utils/AdapterConfigurationSchema"; export class RegisterFormsApi implements RegisterApi { - - /** - * The following publish endpoints (/publish, /published/{id}, /published) - * are used from the designer for operating in 'preview' mode. - * I.E. Designs saved in the designer can be accessed in the runner for viewing. - * The designer also uses these endpoints as a persistence mechanism for storing and retrieving data - * for its own purposes so if you're changing these endpoints you likely need to go and amend - * the designer too! - */ register(server: HapiServer) { - server.route({ - method: "post", - path: "/publish", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {adapterCacheService} = request.services([]); - const payload = request.payload as FormPayload; - const {id, configuration} = payload; - const parsedConfiguration = - typeof configuration === "string" - ? JSON.parse(configuration) - : configuration; - if (parsedConfiguration.configuration) { - await adapterCacheService.setFormConfiguration(id, parsedConfiguration, request.server) - } else { - await adapterCacheService.setFormConfiguration(id, {configuration: parsedConfiguration}, request.server) - } - return h.response({}).code(204); - } - }); - - server.route({ - method: "get", - path: "/published/{id}", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {id} = request.params; - const {adapterCacheService} = request.services([]); - const form = await adapterCacheService.getFormAdapterModel(id, request); - if (!form) { - return h.response({}).code(204); - } - const {values} = await adapterCacheService.getFormAdapterModel(id, request); - return h.response(JSON.stringify({id, values})).code(200); - } - }); - - server.route({ - method: "get", - path: "/published", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {adapterCacheService} = request.services([]); - return h - .response(JSON.stringify(await adapterCacheService.getFormConfigurations(request))) - .code(200); - } - }); - server.route({ method: "get", path: "/", From be27863d2f38f1da7a13c665ca6a01222d5dfc36 Mon Sep 17 00:00:00 2001 From: William May Date: Wed, 10 Sep 2025 16:59:03 +0100 Subject: [PATCH 04/11] FLS-1452 - Remove redundant method getFormConfigurations from AdapterCacheService getFormConfigurations was used within one of the publishing endpoints to send back all of the forms in the cache to Form Designer so it could display them on its existing forms page. This endpoint has been removed from Runner now and Form Designer is being re-pointed at the Pre-Award API to support this requirement. --- .../server/services/AdapterCacheService.ts | 48 ------------------- 1 file changed, 48 deletions(-) diff --git a/runner/src/server/services/AdapterCacheService.ts b/runner/src/server/services/AdapterCacheService.ts index 4bd10fa4..bd4c92e3 100644 --- a/runner/src/server/services/AdapterCacheService.ts +++ b/runner/src/server/services/AdapterCacheService.ts @@ -253,54 +253,6 @@ export class AdapterCacheService extends CacheService { throw Boom.notFound("Cannot find the given form"); } - async getFormConfigurations(request: HapiRequest) { - //@ts-ignore - if (request.server.app.redis) { - return await this.getFormDisplayConfigurationsFromRedisCache(request); - } else { - return await this.getFormConfigurationsFromInMemoryCache(request); - } - - } - - private async getFormConfigurationsFromInMemoryCache(request: HapiRequest) { - const configs: FormConfiguration[] = [] - //@ts-ignore - for (const key of request.server.app.inMemoryFormKeys) { - const configObj = JSON.parse(await this.cache.get(`${key}`)); - const result = AdapterSchema.validate(configObj.configuration, {abortEarly: false}); - configs.push( - new FormConfiguration( - key.replace(FORMS_KEY_PREFIX, ""), - result.value.name, - undefined, - result.value.feedback?.feedbackForm - ) - ) - } - return configs; - } - - private async getFormDisplayConfigurationsFromRedisCache(request: HapiRequest) { - //@ts-ignore - const redisClient: Redis = request.server.app.redis; - const keys = await redisClient.keys(`${FORMS_KEY_PREFIX}*`); - const configs: FormConfiguration[] = [] - for (const key of keys) { - const configObj = JSON.parse(await redisClient.get(`${key}`)); - const result = AdapterSchema.validate(configObj.configuration, {abortEarly: false}); - configs.push( - new FormConfiguration( - key.replace(FORMS_KEY_PREFIX, ""), - result.value.name, - undefined, - result.value.feedback?.feedbackForm - ) - ) - } - return configs; - } - private getRedisClient() { if (redisHost || redisUri) { const redisOptions: { From 8d8bc369a5dd5fe0d1495cb08c9d437ad5494ae2 Mon Sep 17 00:00:00 2001 From: William May Date: Tue, 9 Sep 2025 18:47:49 +0100 Subject: [PATCH 05/11] FLS-1452 - Refactor RegisterFormsApi for readability This is a non-functional change, a pure refactor. We now inline consistently in calls to server.route() (instead of creating variables only used in one place), group middleware functions and place them above route functions, and reduce whitespace. --- .../plugins/engine/api/RegisterFormsApi.ts | 188 ++++++------------ 1 file changed, 66 insertions(+), 122 deletions(-) diff --git a/runner/src/server/plugins/engine/api/RegisterFormsApi.ts b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts index 4432d21f..81225bdc 100644 --- a/runner/src/server/plugins/engine/api/RegisterFormsApi.ts +++ b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts @@ -1,6 +1,5 @@ import {RegisterApi} from "./RegisterApi"; import {HapiRequest, HapiResponseToolkit, HapiServer} from "../../../types"; -import {FormPayload} from "../../../../../../digital-form-builder/runner/src/server/plugins/engine/types"; // @ts-ignore import Boom from "boom"; import {PluginUtil} from "../util/PluginUtil"; @@ -13,29 +12,10 @@ import {config} from "../../utils/AdapterConfigurationSchema"; export class RegisterFormsApi implements RegisterApi { register(server: HapiServer) { - server.route({ - method: "get", - path: "/", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {adapterCacheService} = request.services([]); - const model = await adapterCacheService.getFormAdapterModel("components", request); - if (model) { - return PluginUtil.getStartPageRedirect(request, h, "components", model); - } - if (config.serviceStartPage) { - return h.redirect(config.serviceStartPage); - } - throw Boom.notFound("No default form found"); - } - }); + const {s3UploadService} = server.services([]); - const queryParamPreHandler = async ( - request: HapiRequest, - h: HapiResponseToolkit - ) => { + // Middleware to prepopulate fields from query parameters + const queryParamPreHandler = async (request: HapiRequest, h: HapiResponseToolkit) => { const {query} = request; const {id} = request.params; const {adapterCacheService} = request.services([]); @@ -43,21 +23,13 @@ export class RegisterFormsApi implements RegisterApi { if (!model) { throw Boom.notFound("No form found for id"); } - const prePopFields = model.fieldsForPrePopulation; - if ( - Object.keys(query).length === 0 || - Object.keys(prePopFields).length === 0 - ) { + if (Object.keys(query).length === 0 || Object.keys(prePopFields).length === 0) { return h.continue; } // @ts-ignore const state = await adapterCacheService.getState(request); - const newValues = getValidStateFromQueryParameters( - prePopFields, - query, - state - ); + const newValues = getValidStateFromQueryParameters(prePopFields, query, state); // @ts-ignore await adapterCacheService.mergeState(request, newValues); if (Object.keys(newValues).length > 0) { @@ -66,43 +38,56 @@ export class RegisterFormsApi implements RegisterApi { return h.continue; }; - /** - * Middleware to check if the user session is still valid. - * - * If the session is dropped, it will throw a client timeout error - */ - const checkUserSession = async ( - request: HapiRequest, - h: HapiResponseToolkit - ) => { + // Middleware to check if the user session is still valid + const checkUserSession = async (request: HapiRequest, h: HapiResponseToolkit) => { const {adapterCacheService} = request.services([]); - // @ts-ignore const state = await adapterCacheService.getState(request); - if (config.copilotEnv == "prod" && !state.callback) { - // if you are here the session likely dropped + // If you are here the session likely dropped request.logger.error(["checkUserSession"], `Session expired ${request.yar.id}`); - throw Boom.clientTimeout("Session expired"); } - return h.continue; - } + }; + + // Middleware to handle file uploads + const handleFiles = async (request: HapiRequest, h: HapiResponseToolkit) => { + const {path, id} = request.params; + const {adapterCacheService} = request.services([]); + const model = await adapterCacheService.getFormAdapterModel(id, request); + const page = model?.pages.find( + (page) => PluginUtil.normalisePath(page.path) === PluginUtil.normalisePath(path) + ); + // @ts-ignore + return s3UploadService.handleUploadRequest(request, h, page.pageDef); + }; + + server.route({ + method: "get", + path: "/", + options: { + description: "Default route - redirects to a default form if configured", + }, + handler: async (request: HapiRequest, h: HapiResponseToolkit) => { + const {adapterCacheService} = request.services([]); + const model = await adapterCacheService.getFormAdapterModel("components", request); + if (model) { + return PluginUtil.getStartPageRedirect(request, h, "components", model); + } + if (config.serviceStartPage) { + return h.redirect(config.serviceStartPage); + } + throw Boom.notFound("No default form found"); + } + }); server.route({ method: "get", path: "/{id}", options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - pre: [ - { - method: queryParamPreHandler - }, - { - method: checkUserSession - } - ] + description: "Form start page", + pre: [{method: queryParamPreHandler}, {method: checkUserSession}], }, handler: async (request: HapiRequest, h: HapiResponseToolkit) => { const {id} = request.params; @@ -115,19 +100,13 @@ export class RegisterFormsApi implements RegisterApi { } }); - const getOptions: any = { + server.route({ method: "get", path: "/{id}/{path*}", options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - pre: [ - { - method: queryParamPreHandler - }, - { - method: checkUserSession - } - ], + description: "Form page", + pre: [{method: queryParamPreHandler}, {method: checkUserSession}], + auth: config.jwtAuthEnabled === "true" ? jwtAuthStrategyName : false, }, handler: async (request: HapiRequest, h: HapiResponseToolkit) => { const {path, id} = request.params; @@ -144,55 +123,13 @@ export class RegisterFormsApi implements RegisterApi { } throw Boom.notFound("No form or page found"); } - } - - // TODO: Stop being naughty! Conditionally disabling auth for pre-prod envs is a temporary measure for getting - // FAB into production - if (config.jwtAuthEnabled && config.jwtAuthEnabled === "true") { - getOptions.options.auth = jwtAuthStrategyName - } - - server.route(getOptions); - - const {s3UploadService} = server.services([]); - - const handleFiles = async (request: HapiRequest, h: HapiResponseToolkit) => { - const {path, id} = request.params; - const {adapterCacheService} = request.services([]); - const model = await adapterCacheService.getFormAdapterModel(id, request); - const page = model?.pages.find( - (page) => PluginUtil.normalisePath(page.path) === PluginUtil.normalisePath(path) - ); - // @ts-ignore - return s3UploadService.handleUploadRequest(request, h, page.pageDef); - }; - - const postHandler = async ( - request: HapiRequest, - h: HapiResponseToolkit - ) => { - const {path, id} = request.params; - const {adapterCacheService} = request.services([]); - const model = await adapterCacheService.getFormAdapterModel(id, request); - - if (model) { - const page = model.pages.find( - (page) => page.path.replace(/^\//, "") === path.replace(/^\//, "") - ); - - if (page) { - return page.makePostRouteHandler()(request, h); - } - } - - throw Boom.notFound("No form of path found"); - }; + }); - let postConfig: any = { + server.route({ method: "post", path: "/{id}/{path*}", options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", + description: "Form submission", plugins: { "hapi-rate-limit": { userPathLimit: 10 @@ -210,15 +147,22 @@ export class RegisterFormsApi implements RegisterApi { } }, pre: [{method: handleFiles}], - handler: postHandler, - } - } - if (config.jwtAuthEnabled && config.jwtAuthEnabled === "true") { - postConfig.options.auth = jwtAuthStrategyName - } - server.route(postConfig); - + auth: config.jwtAuthEnabled === "true" ? jwtAuthStrategyName : false, + }, + handler: async (request: HapiRequest, h: HapiResponseToolkit) => { + const {path, id} = request.params; + const {adapterCacheService} = request.services([]); + const model = await adapterCacheService.getFormAdapterModel(id, request); + if (model) { + const page = model.pages.find( + (page) => page.path.replace(/^\//, "") === path.replace(/^\//, "") + ); + if (page) { + return page.makePostRouteHandler()(request, h); + } + } + throw Boom.notFound("No form or path found"); + }, + }); } - - } From 946afd47af8a79f46e6d463db272388a97579a6f Mon Sep 17 00:00:00 2001 From: William May Date: Thu, 11 Sep 2025 19:41:33 +0100 Subject: [PATCH 06/11] FLS-1452 - Simplify code in AdapterCacheService around Redis client Currently we have duplicate Redis client configuration code in the private AdapterCacheService method getRedisClient and in the catboxProvider function. The way the code is formatted is also unnecessarily space-consuming. This commit creates a new shared utility function createRedisClient which can be used in both places, and uses concise formatting to reduce lines of code significantly. --- .../server/services/AdapterCacheService.ts | 114 +++++------------- 1 file changed, 32 insertions(+), 82 deletions(-) diff --git a/runner/src/server/services/AdapterCacheService.ts b/runner/src/server/services/AdapterCacheService.ts index bd4c92e3..f0a60402 100644 --- a/runner/src/server/services/AdapterCacheService.ts +++ b/runner/src/server/services/AdapterCacheService.ts @@ -42,6 +42,22 @@ enum ADDITIONAL_IDENTIFIER { Confirmation = ":confirmation", } +const createRedisClient = (): Redis | null => { + if (redisHost || redisUri) { + const redisOptions: {password?: string; tls?: {};} = {}; + if (redisPassword) redisOptions.password = redisPassword; + if (redisTls) redisOptions.tls = {}; + + return isSingleRedis + ? new Redis(redisUri ?? {host: redisHost, port: redisPort, password: redisPassword}) + : new Redis.Cluster( + [{host: redisHost, port: redisPort}], + {dnsLookup: (address, callback) => callback(null, address, 4), redisOptions} + ); + } + return null; +}; + export class AdapterCacheService extends CacheService { constructor(server: HapiServer) { @@ -253,49 +269,16 @@ export class AdapterCacheService extends CacheService { throw Boom.notFound("Cannot find the given form"); } - private getRedisClient() { - if (redisHost || redisUri) { - const redisOptions: { - password?: string; - tls?: {}; - } = {}; - - if (redisPassword) { - redisOptions.password = redisPassword; - } - - if (redisTls) { - redisOptions.tls = {}; - } - - const client = isSingleRedis - ? new Redis( - redisUri ?? { - host: redisHost, - port: redisPort, - password: redisPassword, - } - ) - : new Redis.Cluster( - [ - { - host: redisHost, - port: redisPort, - }, - ], - { - dnsLookup: (address, callback) => callback(null, address, 4), - redisOptions, - } - ); - return client; - } else { - console.log({ - ...LOGGER_DATA, - message: `[FORM-CACHE] using memory caching`, - }) - } - } + private getRedisClient(): Redis | null { + const client = createRedisClient(); + if (!client) { + console.log({ + ...LOGGER_DATA, + message: `[FORM-CACHE] using memory caching`, + }); + } + return client; + } } export const catboxProvider = () => { @@ -305,49 +288,16 @@ export const catboxProvider = () => { */ const provider = { constructor: redisHost || redisUri ? CatboxRedis.Engine : CatboxMemory.Engine, - options: {}, + options: {partition}, }; if (redisHost || redisUri) { - console.log("Starting redis session management") - const redisOptions: { - password?: string; - tls?: {}; - } = {}; - - if (redisPassword) { - redisOptions.password = redisPassword; - } - - if (redisTls) { - redisOptions.tls = {}; - } - - const client = isSingleRedis - ? new Redis( - redisUri ?? { - host: redisHost, - port: redisPort, - password: redisPassword, - } - ) - : new Redis.Cluster( - [ - { - host: redisHost, - port: redisPort, - }, - ], - { - dnsLookup: (address, callback) => callback(null, address, 4), - redisOptions, - } - ); - provider.options = {client, partition}; - console.log(`Redis Url : ${redisUri} session management`); + console.log("Starting redis session management"); + const client = createRedisClient(); + provider.options = {client, partition}; + console.log(`Redis Url : ${redisUri} session management`); } else { - console.log("Starting in memory session management") - provider.options = {partition}; + console.log("Starting in memory session management"); } return provider; From d30c4887d236929b8b95462f59d3f35de5bd0508 Mon Sep 17 00:00:00 2001 From: William May Date: Thu, 11 Sep 2025 19:47:31 +0100 Subject: [PATCH 07/11] FLS-1452 - Abstract Redis and Catbox cache clients in AdapterCacheService This commit introduces a common interface attribute formStorage on the AdapterCacheService to represent either a Redis client or a Catbox (in-memory) cache client. This enables us to save significant lines of code, where previously we branched off to different functions depending on whether we were using Redis or not, now we can concentrate logic in fewer functions with no branching. We're able to do this because the way we use the two clients is almost identical since we no longer need getKeys functionality. Previously, because the Catbox cache client doesn't have a getKeys utility built-in like Redis, special code was required to keep track of the keys in the cache in the case of Catbox. --- .../src/server/plugins/engine/MainPlugin.ts | 2 +- .../server/services/AdapterCacheService.ts | 135 ++++-------------- 2 files changed, 32 insertions(+), 105 deletions(-) diff --git a/runner/src/server/plugins/engine/MainPlugin.ts b/runner/src/server/plugins/engine/MainPlugin.ts index 1378fc07..032cb026 100644 --- a/runner/src/server/plugins/engine/MainPlugin.ts +++ b/runner/src/server/plugins/engine/MainPlugin.ts @@ -18,7 +18,7 @@ export const plugin = { let countError = 0; for (const config of configs) { try { - await adapterCacheService.setFormConfiguration(config.id, config, server); + await adapterCacheService.setFormConfiguration(config.id, config); countOk++; } catch (e) { countError++; diff --git a/runner/src/server/services/AdapterCacheService.ts b/runner/src/server/services/AdapterCacheService.ts index f0a60402..a075d368 100644 --- a/runner/src/server/services/AdapterCacheService.ts +++ b/runner/src/server/services/AdapterCacheService.ts @@ -15,8 +15,6 @@ import Crypto from 'crypto'; import {HapiRequest, HapiServer} from "../types"; import {AdapterFormModel} from "../plugins/engine/models"; import Boom from "boom"; -import {FormConfiguration} from "@xgovformbuilder/model"; -import {AdapterSchema} from "@communitiesuk/model"; const partition = "cache"; const LOGGER_DATA = { @@ -59,22 +57,26 @@ const createRedisClient = (): Redis | null => { }; export class AdapterCacheService extends CacheService { + private formStorage: Redis | any; constructor(server: HapiServer) { //@ts-ignore super(server); - //@ts-ignore - server.app.redis = this.getRedisClient() - //@ts-ignore - if (!server.app.redis) { - // starting up the in memory cache + const redisClient = this.getRedisClient(); + if (redisClient) { + this.formStorage = redisClient; + } else { + // Starting up the in memory cache this.cache.client.start(); - //@ts-ignore - server.app.inMemoryFormKeys = [] + this.formStorage = { + get: (key) => this.cache.get(key), + set: (key, value) => this.cache.set(key, value, {expiresIn: 0}), + setex: (key, ttl, value) => this.cache.set(key, value, {expiresIn: ttl}), + } } } - async activateSession(jwt, request) { + async activateSession(jwt, request): Promise<{ redirectPath: string }> { request.logger.info(`[ACTIVATE-SESSION] jwt ${jwt}`); const initialisedSession = await this.cache.get(this.JWTKey(jwt)); request.logger.info(`[ACTIVATE-SESSION] session details ${initialisedSession}`); @@ -83,14 +85,12 @@ export class AdapterCacheService extends CacheService { const userSessionKey = {segment: partition, id: `${request.yar.id}:${payload.group}`}; request.logger.info(`[ACTIVATE-SESSION] session metadata ${userSessionKey}`); const {redirectPath} = await super.activateSession(jwt, request); - let redirectPathNew = redirectPath const form_session_identifier = initialisedSession.metadata?.form_session_identifier; if (form_session_identifier) { userSessionKey.id = `${userSessionKey.id}:${form_session_identifier}`; redirectPathNew = `${redirectPathNew}?form_session_identifier=${form_session_identifier}`; } - if (config.overwriteInitialisedSession) { request.logger.info("[ACTIVATE-SESSION] Replacing user session with initialisedSession"); this.cache.set(userSessionKey, initialisedSession, sessionTimeout); @@ -120,7 +120,7 @@ export class AdapterCacheService extends CacheService { * @param additionalIdentifier - appended to the id */ //@ts-ignore - Key(request: HapiRequest, additionalIdentifier?: ADDITIONAL_IDENTIFIER) { + Key(request: HapiRequest, additionalIdentifier?: ADDITIONAL_IDENTIFIER): { segment: string; id: string } { let id = `${request.yar.id}:${request.params.id}`; if (request.query.form_session_identifier) { @@ -141,43 +141,33 @@ export class AdapterCacheService extends CacheService { * @param configuration form definition configurations * @param server server object */ - async setFormConfiguration(formId: string, configuration: any, server: HapiServer) { - if (formId && configuration) { - //@ts-ignore - if (server.app.redis) { - await this.addConfigurationsToRedisCache(server, configuration, formId); - } else { - await this.addConfigurationIntoInMemoryCache(configuration, formId, server); - } - } - } - - private async addConfigurationIntoInMemoryCache(configuration: any, formId: string, server: HapiServer) { - const hashValue = Crypto.createHash('sha256').update(JSON.stringify(configuration)).digest('hex') + async setFormConfiguration(formId: string, configuration: any): Promise { + if (!formId || !configuration) return; + const hashValue = Crypto.createHash('sha256') + .update(JSON.stringify(configuration)) + .digest('hex'); + const key = `${FORMS_KEY_PREFIX}${formId}`; try { - const jsonDataString = await this.cache.get(`${FORMS_KEY_PREFIX}${formId}`); - if (jsonDataString === null) { - // Adding new config into redis cache service with the hash value + const existingConfigString = await this.formStorage.get(key); + if (existingConfigString === null) { + // Adding new config with the hash value const stringConfig = JSON.stringify({ ...configuration, id: configuration.id, hash: hashValue }); - //@ts-ignore - server.app.inMemoryFormKeys.push(`${FORMS_KEY_PREFIX}${formId}`) - // Adding data into redis cache - await this.cache.set(`${FORMS_KEY_PREFIX}${formId}`, stringConfig, {expiresIn: 0}); + await this.formStorage.set(key, stringConfig); } else { - // Redis has the data and gets current data set to check hash - const configObj = JSON.parse(jsonDataString); - if (configObj && configObj.hash && hashValue !== configObj.hash) { - // if hash function is change then updating the configuration + // Check if hash has changed + const existingConfig = JSON.parse(existingConfigString); + if (existingConfig?.hash !== hashValue) { + // Hash has changed, update the configuration const stringConfig = JSON.stringify({ ...configuration, id: configuration.id, hash: hashValue }); - await this.cache.set(`${FORMS_KEY_PREFIX}${formId}`, stringConfig, {expiresIn: 0}); + await this.formStorage.set(key, stringConfig); } } } catch (error) { @@ -185,73 +175,10 @@ export class AdapterCacheService extends CacheService { } } - private async addConfigurationsToRedisCache(server: HapiServer, configuration: any, formId: string) { - //@ts-ignore - const redisClient: Redis = server.app.redis - const hashValue = Crypto.createHash('sha256').update(JSON.stringify(configuration)).digest('hex') - if (redisClient) { - const jsonDataString = await redisClient.get(`${FORMS_KEY_PREFIX}${formId}`); - if (jsonDataString === null) { - // Adding new config into redis cache service with the hash value - const stringConfig = JSON.stringify({ - ...configuration, - id: configuration.id, - hash: hashValue - }); - // Adding data into redis cache - await redisClient.set(`${FORMS_KEY_PREFIX}${formId}`, stringConfig); - } else { - // Redis has the data and gets current data set to check hash - const configObj = JSON.parse(jsonDataString); - if (configObj && configObj.hash && hashValue !== configObj.hash) { - // if hash function is change then updating the configuration - const stringConfig = JSON.stringify({ - ...configuration, - id: configuration.id, - hash: hashValue - }); - await redisClient.set(`${FORMS_KEY_PREFIX}${formId}`, stringConfig); - } - } - } - } - - async getFormAdapterModel(formId: string, request: HapiRequest) { - //@ts-ignore - if (request.server.app.redis) { - return await this.getConfigurationFromRedisCache(request, formId); - } else { - return await this.getConfigurationFromInMemoryCache(request, formId); - } - } - - private async getConfigurationFromInMemoryCache(request: HapiRequest, formId: string) { - const {translationLoaderService} = request.services([]); - const translations = translationLoaderService.getTranslations(); - const jsonDataString = await this.cache.get(`${FORMS_KEY_PREFIX}${formId}`); - if (jsonDataString !== null) { - const configObj = JSON.parse(jsonDataString); - return new AdapterFormModel(configObj.configuration, { - basePath: configObj.id ? configObj.id : formId, - hash: configObj.hash, - previewMode: true, - translationEn: translations.en, - translationCy: translations.cy - }) - } - request.logger.error({ - ...LOGGER_DATA, - message: `[FORM-CACHE] Cannot find the form ${formId}` - }); - throw Boom.notFound("Cannot find the given form"); - } - - private async getConfigurationFromRedisCache(request: HapiRequest, formId: string) { - //@ts-ignore - const redisClient: Redis = request.server.app.redis + async getFormAdapterModel(formId: string, request: HapiRequest): Promise { const {translationLoaderService} = request.services([]); const translations = translationLoaderService.getTranslations(); - const jsonDataString = await redisClient.get(`${FORMS_KEY_PREFIX}${formId}`); + const jsonDataString = await this.formStorage.get(`${FORMS_KEY_PREFIX}${formId}`); if (jsonDataString !== null) { const configObj = JSON.parse(jsonDataString); return new AdapterFormModel(configObj.configuration, { @@ -260,7 +187,7 @@ export class AdapterCacheService extends CacheService { previewMode: true, translationEn: translations.en, translationCy: translations.cy - }) + }); } request.logger.error({ ...LOGGER_DATA, From 940503e5a8d36bf5f7df56ce527feb04f8fb18af Mon Sep 17 00:00:00 2001 From: William May Date: Thu, 11 Sep 2025 11:21:29 +0100 Subject: [PATCH 08/11] FLS-1452 - Stop loading forms from fsd_config into cache on app startup We are going to fetch forms from the Pre-Award API instead. We need to keep the loading of specific forms to ensure unit tests pass. --- .../server/plugins/ConfigureFormsPlugin.ts | 5 +-- .../service/ConfigurationFormsService.ts | 31 ------------------- 2 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 runner/src/server/plugins/engine/service/ConfigurationFormsService.ts diff --git a/runner/src/server/plugins/ConfigureFormsPlugin.ts b/runner/src/server/plugins/ConfigureFormsPlugin.ts index 242ee604..e27963d7 100644 --- a/runner/src/server/plugins/ConfigureFormsPlugin.ts +++ b/runner/src/server/plugins/ConfigureFormsPlugin.ts @@ -1,7 +1,6 @@ import path from "path"; import {plugin} from "./engine/MainPlugin"; -import {loadForms} from "./engine/service/ConfigurationFormsService"; import {idFromFilename} from "../../../../digital-form-builder/runner/src/server/plugins/engine/helpers"; import { FormConfiguration @@ -15,7 +14,7 @@ const relativeTo = __dirname; export const ConfigureFormsPlugin: ConfigureEnginePluginType = ( formFileName, formFilePath, options?: EngineOptions) => { - let configs: FormConfiguration[]; + let configs: FormConfiguration[] = []; if (formFileName && formFilePath) { configs = [ @@ -24,8 +23,6 @@ export const ConfigureFormsPlugin: ConfigureEnginePluginType = ( id: idFromFilename(formFileName) } ]; - } else { - configs = loadForms(); } const modelOptions = { diff --git a/runner/src/server/plugins/engine/service/ConfigurationFormsService.ts b/runner/src/server/plugins/engine/service/ConfigurationFormsService.ts deleted file mode 100644 index febe20ab..00000000 --- a/runner/src/server/plugins/engine/service/ConfigurationFormsService.ts +++ /dev/null @@ -1,31 +0,0 @@ -import fs from "fs"; -import path from "path"; - -import {idFromFilename} from "../../../../../../digital-form-builder/runner/src/server/plugins/engine/helpers"; -import { - FormConfiguration -} from "../../../../../../digital-form-builder/runner/src/server/plugins/engine/services/configurationService"; - -const FORMS_FOLDER = path.join(__dirname, "..", "..", "..", "forms"); - -/** - * Reads the runner/src/server/forms directory for JSON files. The forms that are found will be loaded up at localhost:3009/id - */ -export const loadForms = (): FormConfiguration[] => { - const configFiles = fs - .readdirSync(FORMS_FOLDER) - .filter((filename: string) => filename.indexOf(".json") >= 0); - - // @ts-ignore - return configFiles.map((configFile) => { - const dataFilePath = path.join(FORMS_FOLDER, configFile); - try { - const configuration = require(dataFilePath); - const id = idFromFilename(configFile); - return {configuration, id}; - } catch (e) { - console.error(`Failed to load configuration: filename: [${dataFilePath}] ${e}`); - } - }); -}; - From 2588a78503ed1433f3024bf2017ef31d7f987056 Mon Sep 17 00:00:00 2001 From: William May Date: Wed, 10 Sep 2025 16:49:07 +0100 Subject: [PATCH 09/11] FLS-1452 - Create and register new PreAwardApiService This new service facilitates calls to endpoints in the Pre-Award API - the GET /forms/{name}/hash endpoint, and the GET /forms/{name}/published endpoint. --- copilot/fsd-form-runner-adapter/manifest.yml | 1 + .../config/custom-environment-variables.json | 3 +- runner/config/default.js | 2 + runner/src/server/index.ts | 2 + .../src/server/services/PreAwardApiService.ts | 111 ++++++++++++++++++ runner/src/server/types.ts | 2 + 6 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 runner/src/server/services/PreAwardApiService.ts diff --git a/copilot/fsd-form-runner-adapter/manifest.yml b/copilot/fsd-form-runner-adapter/manifest.yml index f551af55..cf57e7b8 100644 --- a/copilot/fsd-form-runner-adapter/manifest.yml +++ b/copilot/fsd-form-runner-adapter/manifest.yml @@ -78,6 +78,7 @@ secrets: RSA256_PUBLIC_KEY_BASE64: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/RSA256_PUBLIC_KEY_BASE64 SESSION_COOKIE_PASSWORD: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/SESSION_COOKIE_PASSWORD INITIALISED_SESSION_KEY: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/SECRET_KEY + FORM_STORE_API_HOST: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/FSD_PRE_AWARD_FORM_STORE_API_HOST # You can override any of the values defined above by environment. environments: diff --git a/runner/config/custom-environment-variables.json b/runner/config/custom-environment-variables.json index 4952c951..cc48c647 100644 --- a/runner/config/custom-environment-variables.json +++ b/runner/config/custom-environment-variables.json @@ -69,5 +69,6 @@ "sentryDsn": "SENTRY_DSN", "sentryTracesSampleRate": "SENTRY_TRACES_SAMPLE_RATE", "copilotEnv": "COPILOT_ENV", - "enableVirusScan": "ENABLE_VIRUS_SCAN" + "enableVirusScan": "ENABLE_VIRUS_SCAN", + "formStoreApiHost": "FORM_STORE_API_HOST" } diff --git a/runner/config/default.js b/runner/config/default.js index f43bb6aa..25c60bd7 100644 --- a/runner/config/default.js +++ b/runner/config/default.js @@ -114,4 +114,6 @@ module.exports = { copilotEnv: "", enableVirusScan: false, + + formStoreApiHost: "https://api.communities.gov.localhost:4004/forms" }; diff --git a/runner/src/server/index.ts b/runner/src/server/index.ts index 8656777e..ac854642 100644 --- a/runner/src/server/index.ts +++ b/runner/src/server/index.ts @@ -40,6 +40,7 @@ import LanguagePlugin from "./plugins/LanguagePlugin"; import {TranslationLoaderService} from "./services/TranslationLoaderService"; import {WebhookService} from "./services/WebhookService"; import {pluginLog} from "./plugins/logging"; +import { PreAwardApiService } from "./services/PreAwardApiService"; const Sentry = require('@sentry/node'); @@ -133,6 +134,7 @@ async function createServer(routeConfig: RouteConfig) { await server.register(pluginAuth); await server.register(LanguagePlugin); + server.registerService([PreAwardApiService]); server.registerService([AdapterCacheService, NotifyService, PayService, WebhookService, AddressService, TranslationLoaderService]); if (config.isE2EModeEnabled && config.isE2EModeEnabled == "true") { console.log("E2E Mode enabled") diff --git a/runner/src/server/services/PreAwardApiService.ts b/runner/src/server/services/PreAwardApiService.ts new file mode 100644 index 00000000..7b276ad4 --- /dev/null +++ b/runner/src/server/services/PreAwardApiService.ts @@ -0,0 +1,111 @@ +import { HapiServer } from "../types"; +import { config } from "../plugins/utils/AdapterConfigurationSchema"; +import Boom from "boom"; +import wreck from "@hapi/wreck"; + +export interface FormHashResponse { + hash: string; +} + +export interface PublishedFormResponse { + configuration: any; + hash: string; +} + +const LOGGER_DATA = { + class: "PreAwardApiService", +} + +export class PreAwardApiService { + private logger: any; + private apiBaseUrl: string; + private wreck: typeof wreck; + + constructor(server: HapiServer) { + this.logger = server.logger; + this.apiBaseUrl = config.formStoreApiHost; + this.wreck = wreck.defaults({ + timeout: 10000, + headers: { + 'accept': 'application/json', + 'content-type': 'application/json' + } + }); + this.logger.info({ + ...LOGGER_DATA, + message: `Service initialized with base URL: ${this.apiBaseUrl}` + }); + } + + /** + * Fetches the published form data including hash for a specific form. + * This is the primary method used by the cache service when a form + * is requested by a user. It only returns forms that have been + * explicitly published in the Pre-Award system. + */ + async getPublishedForm(name: string): Promise { + const url = `${this.apiBaseUrl}/${name}/published`; + this.logger.info({ + ...LOGGER_DATA, + message: `Fetching published form: ${name}`, + url: url + }); + try { + const { payload } = await this.wreck.get(url, {json: true}); + this.logger.info({ + ...LOGGER_DATA, + message: `Successfully fetched published form: ${name}` + }); + return payload as PublishedFormResponse; + } catch (error: any) { + // Handle 404 - form doesn't exist or isn't published + if (error.output?.statusCode === 404) { + this.logger.info({ + ...LOGGER_DATA, + message: `Form ${name} not found or not published in Pre-Award API` + }); + return null; + } + // Handle other errors (network, timeout, server errors) + this.logger.error({ + ...LOGGER_DATA, + message: `Failed to fetch published form ${name}`, + error: error.message + }); + // Don't expose internal error details to the client + throw Boom.serverUnavailable('Pre-Award API is temporarily unavailable'); + } + } + + /** + * Fetches just the hash of a published form. + * This lightweight endpoint allows us to validate our cache without + * downloading the entire form definition. We use this for periodic + * cache freshness checks. + */ + async getFormHash(name: string): Promise { + const url = `${this.apiBaseUrl}/${name}/hash`; + try { + const { payload } = await this.wreck.get(url, {json: true}); + const data = payload as FormHashResponse; + this.logger.debug({ + ...LOGGER_DATA, + message: `Retrieved hash for form ${name}` + }); + return data.hash; + } catch (error: any) { + if (error.output?.statusCode === 404) { + // Form doesn't exist or isn't published - this is normal + return null; + } + // For hash validation failures, we don't want to fail the entire request. + // We'll continue using the cached version and try again later. + this.logger.warn({ + ...LOGGER_DATA, + message: `Could not fetch hash for form ${name}, will use cached version`, + error: error.message + }); + return null; + } + } +} \ No newline at end of file diff --git a/runner/src/server/types.ts b/runner/src/server/types.ts index 659ff259..82309826 100644 --- a/runner/src/server/types.ts +++ b/runner/src/server/types.ts @@ -17,6 +17,7 @@ import {AdapterCacheService, S3UploadService} from "./services"; import {AdapterStatusService} from "./services"; import {WebhookService} from "./services/WebhookService"; import {TranslationLoaderService} from "./services/TranslationLoaderService"; +import { PreAwardApiService } from "./services/PreAwardApiService"; export type ChangeRequest = { @@ -32,6 +33,7 @@ export type Services = (services: string[]) => { webhookService: WebhookService; adapterStatusService: AdapterStatusService; translationLoaderService: TranslationLoaderService; + preAwardApiService: PreAwardApiService; }; export type RouteConfig = { From f242094f79b6e0547623c7d92ed3f7621bb97ca1 Mon Sep 17 00:00:00 2001 From: William May Date: Wed, 10 Sep 2025 17:10:57 +0100 Subject: [PATCH 10/11] FLS-1452 - Actually retrieve forms from Pre-Award API All of the work we've done has been leading up to this point! In this commit we introduce two new methods into the AdapterCacheService - validateCachedForm and fetchAndCacheForm. These are then used in a revamped getAdapterFormModel. Now, instead of simply looking in the cache, which is no longer in itself sufficient as we don't preload the cache on app startup, we check the cache for the form, if it's not there we go and fetch it from the Pre-Award API and cache it, and if it IS there, then we check to make sure that it hasn't been updated in Pre-Award by comparing the hashed form configurations. Session management is unaffected. --- .../server/services/AdapterCacheService.ts | 134 +++++++++++++++--- 1 file changed, 114 insertions(+), 20 deletions(-) diff --git a/runner/src/server/services/AdapterCacheService.ts b/runner/src/server/services/AdapterCacheService.ts index a075d368..c5326147 100644 --- a/runner/src/server/services/AdapterCacheService.ts +++ b/runner/src/server/services/AdapterCacheService.ts @@ -15,6 +15,7 @@ import Crypto from 'crypto'; import {HapiRequest, HapiServer} from "../types"; import {AdapterFormModel} from "../plugins/engine/models"; import Boom from "boom"; +import { PreAwardApiService, PublishedFormResponse } from "./PreAwardApiService"; const partition = "cache"; const LOGGER_DATA = { @@ -34,7 +35,7 @@ if (process.env.FORM_RUNNER_ADAPTER_REDIS_INSTANCE_URI) { redisUri = process.env.FORM_RUNNER_ADAPTER_REDIS_INSTANCE_URI; } -export const FORMS_KEY_PREFIX = "forms:cache:" +export const FORMS_KEY_PREFIX = "forms" enum ADDITIONAL_IDENTIFIER { Confirmation = ":confirmation", @@ -57,11 +58,13 @@ const createRedisClient = (): Redis | null => { }; export class AdapterCacheService extends CacheService { + private apiService: PreAwardApiService; private formStorage: Redis | any; constructor(server: HapiServer) { //@ts-ignore super(server); + this.apiService = server.services([]).preAwardApiService; const redisClient = this.getRedisClient(); if (redisClient) { this.formStorage = redisClient; @@ -134,19 +137,55 @@ export class AdapterCacheService extends CacheService { } /** - * handling form configuration's in a redis cache to easily distribute them among instance - * * If given fom id is not available, it will generate a hash based on the configuration, And it will be saved in redis cache - * * If hash is change then updating the redis - * @param formId form id - * @param configuration form definition configurations - * @param server server object + * Validates cached form against Pre-Award API. + */ + private async validateCachedForm(formId: string, cachedHash: string, request: HapiRequest): Promise { + try { + const currentHash = await this.apiService.getFormHash(formId); + return currentHash === cachedHash; + } catch (error) { + // If we can't validate, assume cache is valid + request.logger.warn({ + ...LOGGER_DATA, + message: `Could not validate cache for form ${formId}, using cached version` + }); + return true; + } + } + + /** + * Fetches form from Pre-Award API and caches it. + */ + private async fetchAndCacheForm(formId: string, request: HapiRequest): Promise { + try { + const apiResponse = await this.apiService.getPublishedForm(formId); + if (!apiResponse) return null; + const formsCacheKey = `${FORMS_KEY_PREFIX}:${formId}`; + await this.formStorage.set(formsCacheKey, JSON.stringify(apiResponse)); + request.logger.info({ + ...LOGGER_DATA, + message: `Cached form ${formId} from Pre-Award API` + }); + return apiResponse as PublishedFormResponse; + } catch (error) { + request.logger.error({ + ...LOGGER_DATA, + message: `Failed to fetch form ${formId}`, + error: error + }); + return null; + } + } + + /** + * This is used to ensure unit tests can populate the cache */ async setFormConfiguration(formId: string, configuration: any): Promise { if (!formId || !configuration) return; const hashValue = Crypto.createHash('sha256') .update(JSON.stringify(configuration)) .digest('hex'); - const key = `${FORMS_KEY_PREFIX}${formId}`; + const key = `${FORMS_KEY_PREFIX}:${formId}`; try { const existingConfigString = await this.formStorage.get(key); if (existingConfigString === null) { @@ -175,25 +214,80 @@ export class AdapterCacheService extends CacheService { } } + /** + * Retrieves form configuration, either from cache or Pre-Award API. + */ async getFormAdapterModel(formId: string, request: HapiRequest): Promise { const {translationLoaderService} = request.services([]); const translations = translationLoaderService.getTranslations(); - const jsonDataString = await this.formStorage.get(`${FORMS_KEY_PREFIX}${formId}`); + const formCacheKey = `${FORMS_KEY_PREFIX}:${formId}`; + const jsonDataString = await this.formStorage.get(formCacheKey); + // We use a separate key to track if we've validated that this form is up-to-date in this session + // We use yar.id instead of form_session_identifier as form_session_identifier is not present in the first request + const formSessionCacheKey = `${formCacheKey}:${request.yar.id}`; + const sessionValidated = await this.formStorage.get(formSessionCacheKey); + let configObj = null; if (jsonDataString !== null) { - const configObj = JSON.parse(jsonDataString); - return new AdapterFormModel(configObj.configuration, { - basePath: configObj.id ? configObj.id : formId, - hash: configObj.hash, - previewMode: true, - translationEn: translations.en, - translationCy: translations.cy + // Cache hit + request.logger.debug({ + ...LOGGER_DATA, + message: `Cache hit for form ${formId}` + }); + configObj = JSON.parse(jsonDataString); + if (!sessionValidated) { + // Validate cached form once per session + request.logger.debug({ + ...LOGGER_DATA, + message: `First access of form ${formId} in yar session ${request.yar.id}, validating cache` + }); + const isValid = await this.validateCachedForm(formId, configObj.hash, request); + if (!isValid) { + request.logger.info({ + ...LOGGER_DATA, + message: `Cache stale for form ${formId}, fetching fresh version` + }); + const freshConfig = await this.fetchAndCacheForm(formId, request); + if (freshConfig) { + configObj = freshConfig; + } + } else { + request.logger.debug({ + ...LOGGER_DATA, + message: `Cache valid for form ${formId}` + }); + } + } else { + request.logger.debug({ + ...LOGGER_DATA, + message: `Form ${formId} already validated in yar session ${request.yar.id}` + }); + } + } else { + // Cache miss - fetch from Pre-Award API + request.logger.info({ + ...LOGGER_DATA, + message: `Cache miss for form ${formId}, fetching from Pre-Award API` + }); + configObj = await this.fetchAndCacheForm(formId, request); + if (!configObj) { + throw Boom.notFound(`Form '${formId}' not found`); + } + } + if (!sessionValidated) { + // Mark form as validated in this session + request.logger.debug({ + ...LOGGER_DATA, + message: `Marking form ${formId} as validated in yar session ${request.yar.id}` }); + await this.formStorage.setex(formSessionCacheKey, sessionTimeout / 1000, true); } - request.logger.error({ - ...LOGGER_DATA, - message: `[FORM-CACHE] Cannot find the form ${formId}` + return new AdapterFormModel(configObj.configuration, { + basePath: formId, + hash: configObj.hash, + previewMode: true, + translationEn: translations.en, + translationCy: translations.cy }); - throw Boom.notFound("Cannot find the given form"); } private getRedisClient(): Redis | null { From 747ff7027bc395064f20047712175ae30d23dbf1 Mon Sep 17 00:00:00 2001 From: William May Date: Wed, 10 Sep 2025 16:53:42 +0100 Subject: [PATCH 11/11] FLS-1452 - Use server logger in AdapterCacheService to reduce logging noise Using the request logger as we do at the moment introduces the 'req' object into every log message, which is noisy and duplicative, especially given that we log the request path upstream. --- .../server/services/AdapterCacheService.ts | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/runner/src/server/services/AdapterCacheService.ts b/runner/src/server/services/AdapterCacheService.ts index c5326147..6e90bc8a 100644 --- a/runner/src/server/services/AdapterCacheService.ts +++ b/runner/src/server/services/AdapterCacheService.ts @@ -60,11 +60,13 @@ const createRedisClient = (): Redis | null => { export class AdapterCacheService extends CacheService { private apiService: PreAwardApiService; private formStorage: Redis | any; + private logger: any; constructor(server: HapiServer) { //@ts-ignore super(server); this.apiService = server.services([]).preAwardApiService; + this.logger = server.logger; const redisClient = this.getRedisClient(); if (redisClient) { this.formStorage = redisClient; @@ -80,13 +82,13 @@ export class AdapterCacheService extends CacheService { } async activateSession(jwt, request): Promise<{ redirectPath: string }> { - request.logger.info(`[ACTIVATE-SESSION] jwt ${jwt}`); + this.logger.info(`[ACTIVATE-SESSION] jwt ${jwt}`); const initialisedSession = await this.cache.get(this.JWTKey(jwt)); - request.logger.info(`[ACTIVATE-SESSION] session details ${initialisedSession}`); + this.logger.info(`[ACTIVATE-SESSION] session details ${initialisedSession}`); const {decoded} = Jwt.token.decode(jwt); const {payload}: { payload: DecodedSessionToken } = decoded; const userSessionKey = {segment: partition, id: `${request.yar.id}:${payload.group}`}; - request.logger.info(`[ACTIVATE-SESSION] session metadata ${userSessionKey}`); + this.logger.info(`[ACTIVATE-SESSION] session metadata ${userSessionKey}`); const {redirectPath} = await super.activateSession(jwt, request); let redirectPathNew = redirectPath const form_session_identifier = initialisedSession.metadata?.form_session_identifier; @@ -95,7 +97,7 @@ export class AdapterCacheService extends CacheService { redirectPathNew = `${redirectPathNew}?form_session_identifier=${form_session_identifier}`; } if (config.overwriteInitialisedSession) { - request.logger.info("[ACTIVATE-SESSION] Replacing user session with initialisedSession"); + this.logger.info("[ACTIVATE-SESSION] Replacing user session with initialisedSession"); this.cache.set(userSessionKey, initialisedSession, sessionTimeout); } else { const currentSession = await this.cache.get(userSessionKey); @@ -103,12 +105,12 @@ export class AdapterCacheService extends CacheService { ...currentSession, ...initialisedSession, }; - request.logger.info("[ACTIVATE-SESSION] Merging user session with initialisedSession"); + this.logger.info("[ACTIVATE-SESSION] Merging user session with initialisedSession"); this.cache.set(userSessionKey, mergedSession, sessionTimeout); } - request.logger.info(`[ACTIVATE-SESSION] redirect ${redirectPathNew}`); + this.logger.info(`[ACTIVATE-SESSION] redirect ${redirectPathNew}`); const key = this.JWTKey(jwt); - request.logger.info(`[ACTIVATE-SESSION] drop key ${JSON.stringify(key)}`); + this.logger.info(`[ACTIVATE-SESSION] drop key ${JSON.stringify(key)}`); await this.cache.drop(key); return { redirectPath: redirectPathNew, @@ -129,7 +131,7 @@ export class AdapterCacheService extends CacheService { if (request.query.form_session_identifier) { id = `${id}:${request.query.form_session_identifier}`; } - request.logger.info(`[ACTIVATE-SESSION] session key ${id} and segment is ${partition}`); + this.logger.info(`[ACTIVATE-SESSION] session key ${id} and segment is ${partition}`); return { segment: partition, id: `${id}${additionalIdentifier ?? ""}`, @@ -139,13 +141,13 @@ export class AdapterCacheService extends CacheService { /** * Validates cached form against Pre-Award API. */ - private async validateCachedForm(formId: string, cachedHash: string, request: HapiRequest): Promise { + private async validateCachedForm(formId: string, cachedHash: string): Promise { try { const currentHash = await this.apiService.getFormHash(formId); return currentHash === cachedHash; } catch (error) { // If we can't validate, assume cache is valid - request.logger.warn({ + this.logger.warn({ ...LOGGER_DATA, message: `Could not validate cache for form ${formId}, using cached version` }); @@ -156,19 +158,19 @@ export class AdapterCacheService extends CacheService { /** * Fetches form from Pre-Award API and caches it. */ - private async fetchAndCacheForm(formId: string, request: HapiRequest): Promise { + private async fetchAndCacheForm(formId: string): Promise { try { const apiResponse = await this.apiService.getPublishedForm(formId); if (!apiResponse) return null; const formsCacheKey = `${FORMS_KEY_PREFIX}:${formId}`; await this.formStorage.set(formsCacheKey, JSON.stringify(apiResponse)); - request.logger.info({ + this.logger.info({ ...LOGGER_DATA, message: `Cached form ${formId} from Pre-Award API` }); return apiResponse as PublishedFormResponse; } catch (error) { - request.logger.error({ + this.logger.error({ ...LOGGER_DATA, message: `Failed to fetch form ${formId}`, error: error @@ -229,53 +231,53 @@ export class AdapterCacheService extends CacheService { let configObj = null; if (jsonDataString !== null) { // Cache hit - request.logger.debug({ + this.logger.debug({ ...LOGGER_DATA, message: `Cache hit for form ${formId}` }); configObj = JSON.parse(jsonDataString); if (!sessionValidated) { // Validate cached form once per session - request.logger.debug({ + this.logger.debug({ ...LOGGER_DATA, message: `First access of form ${formId} in yar session ${request.yar.id}, validating cache` }); - const isValid = await this.validateCachedForm(formId, configObj.hash, request); + const isValid = await this.validateCachedForm(formId, configObj.hash); if (!isValid) { - request.logger.info({ + this.logger.info({ ...LOGGER_DATA, message: `Cache stale for form ${formId}, fetching fresh version` }); - const freshConfig = await this.fetchAndCacheForm(formId, request); + const freshConfig = await this.fetchAndCacheForm(formId); if (freshConfig) { configObj = freshConfig; } } else { - request.logger.debug({ + this.logger.debug({ ...LOGGER_DATA, message: `Cache valid for form ${formId}` }); } } else { - request.logger.debug({ + this.logger.debug({ ...LOGGER_DATA, message: `Form ${formId} already validated in yar session ${request.yar.id}` }); } } else { // Cache miss - fetch from Pre-Award API - request.logger.info({ + this.logger.info({ ...LOGGER_DATA, message: `Cache miss for form ${formId}, fetching from Pre-Award API` }); - configObj = await this.fetchAndCacheForm(formId, request); + configObj = await this.fetchAndCacheForm(formId); if (!configObj) { throw Boom.notFound(`Form '${formId}' not found`); } } if (!sessionValidated) { // Mark form as validated in this session - request.logger.debug({ + this.logger.debug({ ...LOGGER_DATA, message: `Marking form ${formId} as validated in yar session ${request.yar.id}` });