diff --git a/.run/RUNNER.run.xml b/.run/RUNNER.run.xml index 219ebb11..c2135770 100644 --- a/.run/RUNNER.run.xml +++ b/.run/RUNNER.run.xml @@ -29,7 +29,6 @@ - diff --git a/copilot/fsd-form-runner-adapter/manifest.yml b/copilot/fsd-form-runner-adapter/manifest.yml index d9ed5e46..cf57e7b8 100644 --- a/copilot/fsd-form-runner-adapter/manifest.yml +++ b/copilot/fsd-form-runner-adapter/manifest.yml @@ -78,13 +78,13 @@ secrets: RSA256_PUBLIC_KEY_BASE64: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/RSA256_PUBLIC_KEY_BASE64 SESSION_COOKIE_PASSWORD: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/SESSION_COOKIE_PASSWORD INITIALISED_SESSION_KEY: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/SECRET_KEY + FORM_STORE_API_HOST: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/FSD_PRE_AWARD_FORM_STORE_API_HOST # You can override any of the values defined above by environment. environments: dev: variables: JWT_AUTH_ENABLED: false - PREVIEW_MODE: true count: spot: 2 sidecars: @@ -107,7 +107,6 @@ environments: test: variables: JWT_AUTH_ENABLED: false - PREVIEW_MODE: true count: spot: 2 sidecars: @@ -130,7 +129,6 @@ environments: uat: variables: JWT_AUTH_ENABLED: false - PREVIEW_MODE: true count: range: 2-4 cooldown: @@ -174,7 +172,6 @@ environments: SERVICE_START_PAGE: "https://apply.access-funding.communities.gov.uk/account" ELIGIBILITY_RESULT_URL: "https://apply.access-funding.communities.gov.uk/eligibility-result" SENTRY_TRACES_SAMPLE_RATE: 0.1 - PREVIEW_MODE: false count: range: 2-4 cooldown: diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index 0c8e5de4..897403d0 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -25,7 +25,6 @@ services: - "3009:3009" environment: - CHOKIDAR_USEPOLLING=true - - PREVIEW_MODE=true - LAST_COMMIT - LAST_TAG - ACCESSIBILITY_STATEMENT_URL=http://localhost:3008/accessibility_statement @@ -43,7 +42,6 @@ services: - SERVICE_START_PAGE=http://localhost:3008/account - SINGLE_REDIS=true - FORM_RUNNER_ADAPTER_REDIS_INSTANCE_URI=redis://redis-data:6379 - - PREVIEW_MODE=true - ENABLE_VIRUS_SCAN=true command: yarn runner start:test logging: diff --git a/docker-compose.yml b/docker-compose.yml index ef1e94de..2b5bccc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,6 @@ services: - "3009:3009" environment: - CHOKIDAR_USEPOLLING=true - - PREVIEW_MODE=true - LAST_COMMIT - LAST_TAG - JWT_AUTH_ENABLED=false diff --git a/runner/config/custom-environment-variables.json b/runner/config/custom-environment-variables.json index 041d478b..cc48c647 100644 --- a/runner/config/custom-environment-variables.json +++ b/runner/config/custom-environment-variables.json @@ -45,7 +45,6 @@ "authClientAuthUrl": "AUTH_CLIENT_AUTH_URL", "authClientTokenUrl": "AUTH_CLIENT_TOKEN_URL", "authClientProfileUrl": "AUTH_CLIENT_PROFILE_URL", - "previewMode": "PREVIEW_MODE", "enforceCsrf": "ENFORCE_CSRF", "savePerPage": "SAVE_PER_PAGE", "awsBucketName": "AWS_BUCKET_NAME", @@ -70,5 +69,6 @@ "sentryDsn": "SENTRY_DSN", "sentryTracesSampleRate": "SENTRY_TRACES_SAMPLE_RATE", "copilotEnv": "COPILOT_ENV", - "enableVirusScan": "ENABLE_VIRUS_SCAN" + "enableVirusScan": "ENABLE_VIRUS_SCAN", + "formStoreApiHost": "FORM_STORE_API_HOST" } diff --git a/runner/config/default.js b/runner/config/default.js index 4049a989..25c60bd7 100644 --- a/runner/config/default.js +++ b/runner/config/default.js @@ -21,7 +21,6 @@ module.exports = { */ port: 3009, env: "development", - previewMode: false, enforceCsrf: true, singleRedis: false, isE2EModeEnabled: false, @@ -115,4 +114,6 @@ module.exports = { copilotEnv: "", enableVirusScan: false, + + formStoreApiHost: "https://api.communities.gov.localhost:4004/forms" }; diff --git a/runner/config/development.json b/runner/config/development.json index be126c86..9c666686 100644 --- a/runner/config/development.json +++ b/runner/config/development.json @@ -1,6 +1,5 @@ { "isTest": true, - "previewMode": true, "enforceCsrf": false, "env": "development" } diff --git a/runner/config/test.json b/runner/config/test.json index 0d143677..3613637a 100644 --- a/runner/config/test.json +++ b/runner/config/test.json @@ -1,7 +1,6 @@ { "safelist": ["webho.ok"], "isTest": true, - "previewMode": true, "enforceCsrf": false, "initialisedSessionKey": "predictable-key", "env": "test" diff --git a/runner/src/server/index.ts b/runner/src/server/index.ts index 8656777e..ac854642 100644 --- a/runner/src/server/index.ts +++ b/runner/src/server/index.ts @@ -40,6 +40,7 @@ import LanguagePlugin from "./plugins/LanguagePlugin"; import {TranslationLoaderService} from "./services/TranslationLoaderService"; import {WebhookService} from "./services/WebhookService"; import {pluginLog} from "./plugins/logging"; +import { PreAwardApiService } from "./services/PreAwardApiService"; const Sentry = require('@sentry/node'); @@ -133,6 +134,7 @@ async function createServer(routeConfig: RouteConfig) { await server.register(pluginAuth); await server.register(LanguagePlugin); + server.registerService([PreAwardApiService]); server.registerService([AdapterCacheService, NotifyService, PayService, WebhookService, AddressService, TranslationLoaderService]); if (config.isE2EModeEnabled && config.isE2EModeEnabled == "true") { console.log("E2E Mode enabled") diff --git a/runner/src/server/plugins/ConfigureFormsPlugin.ts b/runner/src/server/plugins/ConfigureFormsPlugin.ts index 242ee604..e27963d7 100644 --- a/runner/src/server/plugins/ConfigureFormsPlugin.ts +++ b/runner/src/server/plugins/ConfigureFormsPlugin.ts @@ -1,7 +1,6 @@ import path from "path"; import {plugin} from "./engine/MainPlugin"; -import {loadForms} from "./engine/service/ConfigurationFormsService"; import {idFromFilename} from "../../../../digital-form-builder/runner/src/server/plugins/engine/helpers"; import { FormConfiguration @@ -15,7 +14,7 @@ const relativeTo = __dirname; export const ConfigureFormsPlugin: ConfigureEnginePluginType = ( formFileName, formFilePath, options?: EngineOptions) => { - let configs: FormConfiguration[]; + let configs: FormConfiguration[] = []; if (formFileName && formFilePath) { configs = [ @@ -24,8 +23,6 @@ export const ConfigureFormsPlugin: ConfigureEnginePluginType = ( id: idFromFilename(formFileName) } ]; - } else { - configs = loadForms(); } const modelOptions = { diff --git a/runner/src/server/plugins/engine/MainPlugin.ts b/runner/src/server/plugins/engine/MainPlugin.ts index 3ec372e8..032cb026 100644 --- a/runner/src/server/plugins/engine/MainPlugin.ts +++ b/runner/src/server/plugins/engine/MainPlugin.ts @@ -1,6 +1,6 @@ import {Options} from "./types/PluginOptions"; import {HapiServer} from "../../types"; -import {RegisterFormPublishApi} from "./api"; +import {RegisterFormsApi} from "./api"; const LOGGER_DATA = { @@ -18,7 +18,7 @@ export const plugin = { let countError = 0; for (const config of configs) { try { - await adapterCacheService.setFormConfiguration(config.id, config, server); + await adapterCacheService.setFormConfiguration(config.id, config); countOk++; } catch (e) { countError++; @@ -32,6 +32,6 @@ export const plugin = { ...LOGGER_DATA, message: `[FORM-CACHE] number of forms loaded into cache ok[${countOk}] error[${countError}]` }) - new RegisterFormPublishApi().register(server, options); + new RegisterFormsApi().register(server); } }; diff --git a/runner/src/server/plugins/engine/api/RegisterFormPublishApi.ts b/runner/src/server/plugins/engine/api/RegisterFormPublishApi.ts deleted file mode 100644 index 75033c32..00000000 --- a/runner/src/server/plugins/engine/api/RegisterFormPublishApi.ts +++ /dev/null @@ -1,322 +0,0 @@ -import {RegisterApi} from "./RegisterApi"; -import {HapiRequest, HapiResponseToolkit, HapiServer} from "../../../types"; -import {Options} from "../types/PluginOptions"; -import {FormPayload} from "../../../../../../digital-form-builder/runner/src/server/plugins/engine/types"; -// @ts-ignore -import Boom from "boom"; -import {PluginUtil} from "../util/PluginUtil"; -import { - getValidStateFromQueryParameters -} from "../../../../../../digital-form-builder/runner/src/server/plugins/engine/helpers"; -import {PluginSpecificConfiguration} from "@hapi/hapi"; -import {jwtAuthStrategyName} from "../Auth"; -import {config} from "../../utils/AdapterConfigurationSchema"; - -export class RegisterFormPublishApi implements RegisterApi { - - /** - * The following publish endpoints (/publish, /published/{id}, /published) - * are used from the designer for operating in 'preview' mode. - * I.E. Designs saved in the designer can be accessed in the runner for viewing. - * The designer also uses these endpoints as a persistence mechanism for storing and retrieving data - * for its own purposes so if you're changing these endpoints you likely need to go and amend - * the designer too! - */ - register(server: HapiServer, options: Options) { - const {previewMode} = options; - const disabledRouteDetailString = - "A request was made however previewing is disabled. See environment variable details in runner/README.md if this error is not expected."; - - server.route({ - method: "post", - path: "/publish", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {adapterCacheService} = request.services([]); - // @ts-ignore - if (!previewMode || previewMode==="false") { - request.logger.error( - [`POST /publish`, "previewModeError"], - disabledRouteDetailString - ); - throw Boom.forbidden("Publishing is disabled"); - } - const payload = request.payload as FormPayload; - const {id, configuration} = payload; - - const parsedConfiguration = - typeof configuration === "string" - ? JSON.parse(configuration) - : configuration; - if (parsedConfiguration.configuration) { - await adapterCacheService.setFormConfiguration(id, parsedConfiguration, request.server) - } else { - await adapterCacheService.setFormConfiguration(id, {configuration: parsedConfiguration}, request.server) - } - return h.response({}).code(204); - } - }); - - server.route({ - method: "get", - path: "/published/{id}", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {id} = request.params; - // @ts-ignore - if (!previewMode || previewMode==="false") { - request.logger.error( - [`GET /published/${id}`, "previewModeError"], - disabledRouteDetailString - ); - throw Boom.unauthorized("publishing is disabled"); - } - const {adapterCacheService} = request.services([]); - const form = await adapterCacheService.getFormAdapterModel(id, request); - if (!form) { - return h.response({}).code(204); - } - const {values} = await adapterCacheService.getFormAdapterModel(id, request); - return h.response(JSON.stringify({id, values})).code(200); - } - }); - - server.route({ - method: "get", - path: "/published", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {adapterCacheService} = request.services([]); - // @ts-ignore - if (!previewMode || previewMode==="false") { - request.logger.error( - [`GET /published`, "previewModeError"], - disabledRouteDetailString - ); - throw Boom.unauthorized("publishing is disabled."); - } - return h - .response(JSON.stringify(await adapterCacheService.getFormConfigurations(request))) - .code(200); - } - }); - - server.route({ - method: "get", - path: "/", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {adapterCacheService} = request.services([]); - const model = await adapterCacheService.getFormAdapterModel("components", request); - if (model) { - return PluginUtil.getStartPageRedirect(request, h, "components", model); - } - if (config.serviceStartPage) { - return h.redirect(config.serviceStartPage); - } - throw Boom.notFound("No default form found"); - } - }); - - const queryParamPreHandler = async ( - request: HapiRequest, - h: HapiResponseToolkit - ) => { - const {query} = request; - const {id} = request.params; - const {adapterCacheService} = request.services([]); - const model = await adapterCacheService.getFormAdapterModel(id, request); - if (!model) { - throw Boom.notFound("No form found for id"); - } - - const prePopFields = model.fieldsForPrePopulation; - if ( - Object.keys(query).length === 0 || - Object.keys(prePopFields).length === 0 - ) { - return h.continue; - } - // @ts-ignore - const state = await adapterCacheService.getState(request); - const newValues = getValidStateFromQueryParameters( - prePopFields, - query, - state - ); - // @ts-ignore - await adapterCacheService.mergeState(request, newValues); - if (Object.keys(newValues).length > 0) { - h.request.pre.hasPrepopulatedSessionFromQueryParameter = true; - } - return h.continue; - }; - - /** - * Middleware to check if the user session is still valid. - * - * Changes behaviour only when previewMode is FALSE, meaning PRODUCTION - * If the session is dropped, it will throw a client timeout error - */ - const checkUserSession = async ( - request: HapiRequest, - h: HapiResponseToolkit - ) => { - const {adapterCacheService} = request.services([]); - - // @ts-ignore - const state = await adapterCacheService.getState(request); - - // @ts-ignore isNotPreview is always false on production - const isNotPreview = !previewMode || previewMode==="false" - - if (isNotPreview && !state.callback) { - // if you are here the session likely dropped - request.logger.error(["checkUserSession"], `Session expired ${request.yar.id}`); - - throw Boom.clientTimeout("Session expired"); - } - - return h.continue; - } - - server.route({ - method: "get", - path: "/{id}", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - pre: [ - { - method: queryParamPreHandler - }, - { - method: checkUserSession - } - ] - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {id} = request.params; - const {adapterCacheService} = request.services([]); - const model = await adapterCacheService.getFormAdapterModel(id, request); - if (model) { - return PluginUtil.getStartPageRedirect(request, h, id, model); - } - throw Boom.notFound("No form found for id"); - } - }); - - const getOptions: any = { - method: "get", - path: "/{id}/{path*}", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - pre: [ - { - method: queryParamPreHandler - }, - { - method: checkUserSession - } - ], - }, - handler: async (request: HapiRequest, h: HapiResponseToolkit) => { - const {path, id} = request.params; - const {adapterCacheService} = request.services([]); - const model = await adapterCacheService.getFormAdapterModel(id, request); - const page = model?.pages.find( - (page) => PluginUtil.normalisePath(page.path) === PluginUtil.normalisePath(path) - ); - if (page) { - return page.makeGetRouteHandler()(request, h); - } - if (PluginUtil.normalisePath(path) === "" && model) { - return PluginUtil.getStartPageRedirect(request, h, id, model); - } - throw Boom.notFound("No form or page found"); - } - } - - // TODO: Stop being naughty! Conditionally disabling auth for pre-prod envs is a temporary measure for getting - // FAB into production - if (config.jwtAuthEnabled && config.jwtAuthEnabled === "true") { - getOptions.options.auth = jwtAuthStrategyName - } - - server.route(getOptions); - - const {s3UploadService} = server.services([]); - - const handleFiles = async (request: HapiRequest, h: HapiResponseToolkit) => { - const {path, id} = request.params; - const {adapterCacheService} = request.services([]); - const model = await adapterCacheService.getFormAdapterModel(id, request); - const page = model?.pages.find( - (page) => PluginUtil.normalisePath(page.path) === PluginUtil.normalisePath(path) - ); - // @ts-ignore - return s3UploadService.handleUploadRequest(request, h, page.pageDef); - }; - - const postHandler = async ( - request: HapiRequest, - h: HapiResponseToolkit - ) => { - const {path, id} = request.params; - const {adapterCacheService} = request.services([]); - const model = await adapterCacheService.getFormAdapterModel(id, request); - - if (model) { - const page = model.pages.find( - (page) => page.path.replace(/^\//, "") === path.replace(/^\//, "") - ); - - if (page) { - return page.makePostRouteHandler()(request, h); - } - } - - throw Boom.notFound("No form of path found"); - }; - - let postConfig: any = { - method: "post", - path: "/{id}/{path*}", - options: { - description: "See API-README.md file in the runner/src/server/plugins/engine/api", - plugins: { - "hapi-rate-limit": { - userPathLimit: 10 - } - }, - payload: { - output: "stream", - parse: true, - multipart: {output: "stream"}, - maxBytes: s3UploadService.fileSizeLimit, - failAction: async (request: HapiRequest, h: HapiResponseToolkit) => { - // @ts-ignore - request.server.plugins.crumb.generate?.(request, h); - return h.continue; - } - }, - pre: [{method: handleFiles}], - handler: postHandler, - } - } - if (config.jwtAuthEnabled && config.jwtAuthEnabled === "true") { - postConfig.options.auth = jwtAuthStrategyName - } - server.route(postConfig); - - } - - -} diff --git a/runner/src/server/plugins/engine/api/RegisterFormsApi.ts b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts new file mode 100644 index 00000000..81225bdc --- /dev/null +++ b/runner/src/server/plugins/engine/api/RegisterFormsApi.ts @@ -0,0 +1,168 @@ +import {RegisterApi} from "./RegisterApi"; +import {HapiRequest, HapiResponseToolkit, HapiServer} from "../../../types"; +// @ts-ignore +import Boom from "boom"; +import {PluginUtil} from "../util/PluginUtil"; +import { + getValidStateFromQueryParameters +} from "../../../../../../digital-form-builder/runner/src/server/plugins/engine/helpers"; +import {PluginSpecificConfiguration} from "@hapi/hapi"; +import {jwtAuthStrategyName} from "../Auth"; +import {config} from "../../utils/AdapterConfigurationSchema"; + +export class RegisterFormsApi implements RegisterApi { + register(server: HapiServer) { + const {s3UploadService} = server.services([]); + + // Middleware to prepopulate fields from query parameters + const queryParamPreHandler = async (request: HapiRequest, h: HapiResponseToolkit) => { + const {query} = request; + const {id} = request.params; + const {adapterCacheService} = request.services([]); + const model = await adapterCacheService.getFormAdapterModel(id, request); + if (!model) { + throw Boom.notFound("No form found for id"); + } + const prePopFields = model.fieldsForPrePopulation; + if (Object.keys(query).length === 0 || Object.keys(prePopFields).length === 0) { + return h.continue; + } + // @ts-ignore + const state = await adapterCacheService.getState(request); + const newValues = getValidStateFromQueryParameters(prePopFields, query, state); + // @ts-ignore + await adapterCacheService.mergeState(request, newValues); + if (Object.keys(newValues).length > 0) { + h.request.pre.hasPrepopulatedSessionFromQueryParameter = true; + } + return h.continue; + }; + + // Middleware to check if the user session is still valid + const checkUserSession = async (request: HapiRequest, h: HapiResponseToolkit) => { + const {adapterCacheService} = request.services([]); + // @ts-ignore + const state = await adapterCacheService.getState(request); + if (config.copilotEnv == "prod" && !state.callback) { + // If you are here the session likely dropped + request.logger.error(["checkUserSession"], `Session expired ${request.yar.id}`); + throw Boom.clientTimeout("Session expired"); + } + return h.continue; + }; + + // Middleware to handle file uploads + const handleFiles = async (request: HapiRequest, h: HapiResponseToolkit) => { + const {path, id} = request.params; + const {adapterCacheService} = request.services([]); + const model = await adapterCacheService.getFormAdapterModel(id, request); + const page = model?.pages.find( + (page) => PluginUtil.normalisePath(page.path) === PluginUtil.normalisePath(path) + ); + // @ts-ignore + return s3UploadService.handleUploadRequest(request, h, page.pageDef); + }; + + server.route({ + method: "get", + path: "/", + options: { + description: "Default route - redirects to a default form if configured", + }, + handler: async (request: HapiRequest, h: HapiResponseToolkit) => { + const {adapterCacheService} = request.services([]); + const model = await adapterCacheService.getFormAdapterModel("components", request); + if (model) { + return PluginUtil.getStartPageRedirect(request, h, "components", model); + } + if (config.serviceStartPage) { + return h.redirect(config.serviceStartPage); + } + throw Boom.notFound("No default form found"); + } + }); + + server.route({ + method: "get", + path: "/{id}", + options: { + description: "Form start page", + pre: [{method: queryParamPreHandler}, {method: checkUserSession}], + }, + handler: async (request: HapiRequest, h: HapiResponseToolkit) => { + const {id} = request.params; + const {adapterCacheService} = request.services([]); + const model = await adapterCacheService.getFormAdapterModel(id, request); + if (model) { + return PluginUtil.getStartPageRedirect(request, h, id, model); + } + throw Boom.notFound("No form found for id"); + } + }); + + server.route({ + method: "get", + path: "/{id}/{path*}", + options: { + description: "Form page", + pre: [{method: queryParamPreHandler}, {method: checkUserSession}], + auth: config.jwtAuthEnabled === "true" ? jwtAuthStrategyName : false, + }, + handler: async (request: HapiRequest, h: HapiResponseToolkit) => { + const {path, id} = request.params; + const {adapterCacheService} = request.services([]); + const model = await adapterCacheService.getFormAdapterModel(id, request); + const page = model?.pages.find( + (page) => PluginUtil.normalisePath(page.path) === PluginUtil.normalisePath(path) + ); + if (page) { + return page.makeGetRouteHandler()(request, h); + } + if (PluginUtil.normalisePath(path) === "" && model) { + return PluginUtil.getStartPageRedirect(request, h, id, model); + } + throw Boom.notFound("No form or page found"); + } + }); + + server.route({ + method: "post", + path: "/{id}/{path*}", + options: { + description: "Form submission", + plugins: { + "hapi-rate-limit": { + userPathLimit: 10 + } + }, + payload: { + output: "stream", + parse: true, + multipart: {output: "stream"}, + maxBytes: s3UploadService.fileSizeLimit, + failAction: async (request: HapiRequest, h: HapiResponseToolkit) => { + // @ts-ignore + request.server.plugins.crumb.generate?.(request, h); + return h.continue; + } + }, + pre: [{method: handleFiles}], + auth: config.jwtAuthEnabled === "true" ? jwtAuthStrategyName : false, + }, + handler: async (request: HapiRequest, h: HapiResponseToolkit) => { + const {path, id} = request.params; + const {adapterCacheService} = request.services([]); + const model = await adapterCacheService.getFormAdapterModel(id, request); + if (model) { + const page = model.pages.find( + (page) => page.path.replace(/^\//, "") === path.replace(/^\//, "") + ); + if (page) { + return page.makePostRouteHandler()(request, h); + } + } + throw Boom.notFound("No form or path found"); + }, + }); + } +} diff --git a/runner/src/server/plugins/engine/api/index.ts b/runner/src/server/plugins/engine/api/index.ts index 137726a5..562ed52c 100644 --- a/runner/src/server/plugins/engine/api/index.ts +++ b/runner/src/server/plugins/engine/api/index.ts @@ -2,4 +2,4 @@ export {RegisterSessionApi} from "./RegisterSessionApi" export {RegisterApi} from "./RegisterApi" export {RegisterApplicationStatusApi} from "./RegisterApplicationStatusApi" export {RegisterPublicApi} from "./RegisterPublicApi" -export {RegisterFormPublishApi} from "./RegisterFormPublishApi" +export {RegisterFormsApi} from "./RegisterFormsApi" diff --git a/runner/src/server/plugins/engine/service/ConfigurationFormsService.ts b/runner/src/server/plugins/engine/service/ConfigurationFormsService.ts deleted file mode 100644 index febe20ab..00000000 --- a/runner/src/server/plugins/engine/service/ConfigurationFormsService.ts +++ /dev/null @@ -1,31 +0,0 @@ -import fs from "fs"; -import path from "path"; - -import {idFromFilename} from "../../../../../../digital-form-builder/runner/src/server/plugins/engine/helpers"; -import { - FormConfiguration -} from "../../../../../../digital-form-builder/runner/src/server/plugins/engine/services/configurationService"; - -const FORMS_FOLDER = path.join(__dirname, "..", "..", "..", "forms"); - -/** - * Reads the runner/src/server/forms directory for JSON files. The forms that are found will be loaded up at localhost:3009/id - */ -export const loadForms = (): FormConfiguration[] => { - const configFiles = fs - .readdirSync(FORMS_FOLDER) - .filter((filename: string) => filename.indexOf(".json") >= 0); - - // @ts-ignore - return configFiles.map((configFile) => { - const dataFilePath = path.join(FORMS_FOLDER, configFile); - try { - const configuration = require(dataFilePath); - const id = idFromFilename(configFile); - return {configuration, id}; - } catch (e) { - console.error(`Failed to load configuration: filename: [${dataFilePath}] ${e}`); - } - }); -}; - diff --git a/runner/src/server/services/AdapterCacheService.ts b/runner/src/server/services/AdapterCacheService.ts index 4bd10fa4..6e90bc8a 100644 --- a/runner/src/server/services/AdapterCacheService.ts +++ b/runner/src/server/services/AdapterCacheService.ts @@ -15,8 +15,7 @@ import Crypto from 'crypto'; import {HapiRequest, HapiServer} from "../types"; import {AdapterFormModel} from "../plugins/engine/models"; import Boom from "boom"; -import {FormConfiguration} from "@xgovformbuilder/model"; -import {AdapterSchema} from "@communitiesuk/model"; +import { PreAwardApiService, PublishedFormResponse } from "./PreAwardApiService"; const partition = "cache"; const LOGGER_DATA = { @@ -36,47 +35,69 @@ if (process.env.FORM_RUNNER_ADAPTER_REDIS_INSTANCE_URI) { redisUri = process.env.FORM_RUNNER_ADAPTER_REDIS_INSTANCE_URI; } -export const FORMS_KEY_PREFIX = "forms:cache:" +export const FORMS_KEY_PREFIX = "forms" enum ADDITIONAL_IDENTIFIER { Confirmation = ":confirmation", } +const createRedisClient = (): Redis | null => { + if (redisHost || redisUri) { + const redisOptions: {password?: string; tls?: {};} = {}; + if (redisPassword) redisOptions.password = redisPassword; + if (redisTls) redisOptions.tls = {}; + + return isSingleRedis + ? new Redis(redisUri ?? {host: redisHost, port: redisPort, password: redisPassword}) + : new Redis.Cluster( + [{host: redisHost, port: redisPort}], + {dnsLookup: (address, callback) => callback(null, address, 4), redisOptions} + ); + } + return null; +}; + export class AdapterCacheService extends CacheService { + private apiService: PreAwardApiService; + private formStorage: Redis | any; + private logger: any; constructor(server: HapiServer) { //@ts-ignore super(server); - //@ts-ignore - server.app.redis = this.getRedisClient() - //@ts-ignore - if (!server.app.redis) { - // starting up the in memory cache + this.apiService = server.services([]).preAwardApiService; + this.logger = server.logger; + const redisClient = this.getRedisClient(); + if (redisClient) { + this.formStorage = redisClient; + } else { + // Starting up the in memory cache this.cache.client.start(); - //@ts-ignore - server.app.inMemoryFormKeys = [] + this.formStorage = { + get: (key) => this.cache.get(key), + set: (key, value) => this.cache.set(key, value, {expiresIn: 0}), + setex: (key, ttl, value) => this.cache.set(key, value, {expiresIn: ttl}), + } } } - async activateSession(jwt, request) { - request.logger.info(`[ACTIVATE-SESSION] jwt ${jwt}`); + async activateSession(jwt, request): Promise<{ redirectPath: string }> { + this.logger.info(`[ACTIVATE-SESSION] jwt ${jwt}`); const initialisedSession = await this.cache.get(this.JWTKey(jwt)); - request.logger.info(`[ACTIVATE-SESSION] session details ${initialisedSession}`); + this.logger.info(`[ACTIVATE-SESSION] session details ${initialisedSession}`); const {decoded} = Jwt.token.decode(jwt); const {payload}: { payload: DecodedSessionToken } = decoded; const userSessionKey = {segment: partition, id: `${request.yar.id}:${payload.group}`}; - request.logger.info(`[ACTIVATE-SESSION] session metadata ${userSessionKey}`); + this.logger.info(`[ACTIVATE-SESSION] session metadata ${userSessionKey}`); const {redirectPath} = await super.activateSession(jwt, request); - let redirectPathNew = redirectPath const form_session_identifier = initialisedSession.metadata?.form_session_identifier; if (form_session_identifier) { userSessionKey.id = `${userSessionKey.id}:${form_session_identifier}`; redirectPathNew = `${redirectPathNew}?form_session_identifier=${form_session_identifier}`; } - if (config.overwriteInitialisedSession) { - request.logger.info("[ACTIVATE-SESSION] Replacing user session with initialisedSession"); + this.logger.info("[ACTIVATE-SESSION] Replacing user session with initialisedSession"); this.cache.set(userSessionKey, initialisedSession, sessionTimeout); } else { const currentSession = await this.cache.get(userSessionKey); @@ -84,12 +105,12 @@ export class AdapterCacheService extends CacheService { ...currentSession, ...initialisedSession, }; - request.logger.info("[ACTIVATE-SESSION] Merging user session with initialisedSession"); + this.logger.info("[ACTIVATE-SESSION] Merging user session with initialisedSession"); this.cache.set(userSessionKey, mergedSession, sessionTimeout); } - request.logger.info(`[ACTIVATE-SESSION] redirect ${redirectPathNew}`); + this.logger.info(`[ACTIVATE-SESSION] redirect ${redirectPathNew}`); const key = this.JWTKey(jwt); - request.logger.info(`[ACTIVATE-SESSION] drop key ${JSON.stringify(key)}`); + this.logger.info(`[ACTIVATE-SESSION] drop key ${JSON.stringify(key)}`); await this.cache.drop(key); return { redirectPath: redirectPathNew, @@ -104,13 +125,13 @@ export class AdapterCacheService extends CacheService { * @param additionalIdentifier - appended to the id */ //@ts-ignore - Key(request: HapiRequest, additionalIdentifier?: ADDITIONAL_IDENTIFIER) { + Key(request: HapiRequest, additionalIdentifier?: ADDITIONAL_IDENTIFIER): { segment: string; id: string } { let id = `${request.yar.id}:${request.params.id}`; if (request.query.form_session_identifier) { id = `${id}:${request.query.form_session_identifier}`; } - request.logger.info(`[ACTIVATE-SESSION] session key ${id} and segment is ${partition}`); + this.logger.info(`[ACTIVATE-SESSION] session key ${id} and segment is ${partition}`); return { segment: partition, id: `${id}${additionalIdentifier ?? ""}`, @@ -118,232 +139,169 @@ export class AdapterCacheService extends CacheService { } /** - * handling form configuration's in a redis cache to easily distribute them among instance - * * If given fom id is not available, it will generate a hash based on the configuration, And it will be saved in redis cache - * * If hash is change then updating the redis - * @param formId form id - * @param configuration form definition configurations - * @param server server object + * Validates cached form against Pre-Award API. */ - async setFormConfiguration(formId: string, configuration: any, server: HapiServer) { - if (formId && configuration) { - //@ts-ignore - if (server.app.redis) { - await this.addConfigurationsToRedisCache(server, configuration, formId); - } else { - await this.addConfigurationIntoInMemoryCache(configuration, formId, server); - } + private async validateCachedForm(formId: string, cachedHash: string): Promise { + try { + const currentHash = await this.apiService.getFormHash(formId); + return currentHash === cachedHash; + } catch (error) { + // If we can't validate, assume cache is valid + this.logger.warn({ + ...LOGGER_DATA, + message: `Could not validate cache for form ${formId}, using cached version` + }); + return true; } } - private async addConfigurationIntoInMemoryCache(configuration: any, formId: string, server: HapiServer) { - const hashValue = Crypto.createHash('sha256').update(JSON.stringify(configuration)).digest('hex') + /** + * Fetches form from Pre-Award API and caches it. + */ + private async fetchAndCacheForm(formId: string): Promise { try { - const jsonDataString = await this.cache.get(`${FORMS_KEY_PREFIX}${formId}`); - if (jsonDataString === null) { - // Adding new config into redis cache service with the hash value - const stringConfig = JSON.stringify({ - ...configuration, - id: configuration.id, - hash: hashValue - }); - //@ts-ignore - server.app.inMemoryFormKeys.push(`${FORMS_KEY_PREFIX}${formId}`) - // Adding data into redis cache - await this.cache.set(`${FORMS_KEY_PREFIX}${formId}`, stringConfig, {expiresIn: 0}); - } else { - // Redis has the data and gets current data set to check hash - const configObj = JSON.parse(jsonDataString); - if (configObj && configObj.hash && hashValue !== configObj.hash) { - // if hash function is change then updating the configuration - const stringConfig = JSON.stringify({ - ...configuration, - id: configuration.id, - hash: hashValue - }); - await this.cache.set(`${FORMS_KEY_PREFIX}${formId}`, stringConfig, {expiresIn: 0}); - } - } + const apiResponse = await this.apiService.getPublishedForm(formId); + if (!apiResponse) return null; + const formsCacheKey = `${FORMS_KEY_PREFIX}:${formId}`; + await this.formStorage.set(formsCacheKey, JSON.stringify(apiResponse)); + this.logger.info({ + ...LOGGER_DATA, + message: `Cached form ${formId} from Pre-Award API` + }); + return apiResponse as PublishedFormResponse; } catch (error) { - console.log(error); + this.logger.error({ + ...LOGGER_DATA, + message: `Failed to fetch form ${formId}`, + error: error + }); + return null; } } - private async addConfigurationsToRedisCache(server: HapiServer, configuration: any, formId: string) { - //@ts-ignore - const redisClient: Redis = server.app.redis - const hashValue = Crypto.createHash('sha256').update(JSON.stringify(configuration)).digest('hex') - if (redisClient) { - const jsonDataString = await redisClient.get(`${FORMS_KEY_PREFIX}${formId}`); - if (jsonDataString === null) { - // Adding new config into redis cache service with the hash value + /** + * This is used to ensure unit tests can populate the cache + */ + async setFormConfiguration(formId: string, configuration: any): Promise { + if (!formId || !configuration) return; + const hashValue = Crypto.createHash('sha256') + .update(JSON.stringify(configuration)) + .digest('hex'); + const key = `${FORMS_KEY_PREFIX}:${formId}`; + try { + const existingConfigString = await this.formStorage.get(key); + if (existingConfigString === null) { + // Adding new config with the hash value const stringConfig = JSON.stringify({ ...configuration, id: configuration.id, hash: hashValue }); - // Adding data into redis cache - await redisClient.set(`${FORMS_KEY_PREFIX}${formId}`, stringConfig); + await this.formStorage.set(key, stringConfig); } else { - // Redis has the data and gets current data set to check hash - const configObj = JSON.parse(jsonDataString); - if (configObj && configObj.hash && hashValue !== configObj.hash) { - // if hash function is change then updating the configuration + // Check if hash has changed + const existingConfig = JSON.parse(existingConfigString); + if (existingConfig?.hash !== hashValue) { + // Hash has changed, update the configuration const stringConfig = JSON.stringify({ ...configuration, id: configuration.id, hash: hashValue }); - await redisClient.set(`${FORMS_KEY_PREFIX}${formId}`, stringConfig); + await this.formStorage.set(key, stringConfig); } } + } catch (error) { + console.log(error); } } - async getFormAdapterModel(formId: string, request: HapiRequest) { - //@ts-ignore - if (request.server.app.redis) { - return await this.getConfigurationFromRedisCache(request, formId); - } else { - return await this.getConfigurationFromInMemoryCache(request, formId); - } - } - - private async getConfigurationFromInMemoryCache(request: HapiRequest, formId: string) { - const {translationLoaderService} = request.services([]); - const translations = translationLoaderService.getTranslations(); - const jsonDataString = await this.cache.get(`${FORMS_KEY_PREFIX}${formId}`); - if (jsonDataString !== null) { - const configObj = JSON.parse(jsonDataString); - return new AdapterFormModel(configObj.configuration, { - basePath: configObj.id ? configObj.id : formId, - hash: configObj.hash, - previewMode: true, - translationEn: translations.en, - translationCy: translations.cy - }) - } - request.logger.error({ - ...LOGGER_DATA, - message: `[FORM-CACHE] Cannot find the form ${formId}` - }); - throw Boom.notFound("Cannot find the given form"); - } - - private async getConfigurationFromRedisCache(request: HapiRequest, formId: string) { - //@ts-ignore - const redisClient: Redis = request.server.app.redis + /** + * Retrieves form configuration, either from cache or Pre-Award API. + */ + async getFormAdapterModel(formId: string, request: HapiRequest): Promise { const {translationLoaderService} = request.services([]); const translations = translationLoaderService.getTranslations(); - const jsonDataString = await redisClient.get(`${FORMS_KEY_PREFIX}${formId}`); + const formCacheKey = `${FORMS_KEY_PREFIX}:${formId}`; + const jsonDataString = await this.formStorage.get(formCacheKey); + // We use a separate key to track if we've validated that this form is up-to-date in this session + // We use yar.id instead of form_session_identifier as form_session_identifier is not present in the first request + const formSessionCacheKey = `${formCacheKey}:${request.yar.id}`; + const sessionValidated = await this.formStorage.get(formSessionCacheKey); + let configObj = null; if (jsonDataString !== null) { - const configObj = JSON.parse(jsonDataString); - return new AdapterFormModel(configObj.configuration, { - basePath: configObj.id ? configObj.id : formId, - hash: configObj.hash, - previewMode: true, - translationEn: translations.en, - translationCy: translations.cy - }) - } - request.logger.error({ - ...LOGGER_DATA, - message: `[FORM-CACHE] Cannot find the form ${formId}` - }); - throw Boom.notFound("Cannot find the given form"); - } - - async getFormConfigurations(request: HapiRequest) { - //@ts-ignore - if (request.server.app.redis) { - return await this.getFormDisplayConfigurationsFromRedisCache(request); - } else { - return await this.getFormConfigurationsFromInMemoryCache(request); - } - - } - - private async getFormConfigurationsFromInMemoryCache(request: HapiRequest) { - const configs: FormConfiguration[] = [] - //@ts-ignore - for (const key of request.server.app.inMemoryFormKeys) { - const configObj = JSON.parse(await this.cache.get(`${key}`)); - const result = AdapterSchema.validate(configObj.configuration, {abortEarly: false}); - configs.push( - new FormConfiguration( - key.replace(FORMS_KEY_PREFIX, ""), - result.value.name, - undefined, - result.value.feedback?.feedbackForm - ) - ) - } - return configs; - } - - private async getFormDisplayConfigurationsFromRedisCache(request: HapiRequest) { - //@ts-ignore - const redisClient: Redis = request.server.app.redis; - const keys = await redisClient.keys(`${FORMS_KEY_PREFIX}*`); - const configs: FormConfiguration[] = [] - for (const key of keys) { - const configObj = JSON.parse(await redisClient.get(`${key}`)); - const result = AdapterSchema.validate(configObj.configuration, {abortEarly: false}); - configs.push( - new FormConfiguration( - key.replace(FORMS_KEY_PREFIX, ""), - result.value.name, - undefined, - result.value.feedback?.feedbackForm - ) - ) - } - return configs; - } - - private getRedisClient() { - if (redisHost || redisUri) { - const redisOptions: { - password?: string; - tls?: {}; - } = {}; - - if (redisPassword) { - redisOptions.password = redisPassword; - } - - if (redisTls) { - redisOptions.tls = {}; - } - - const client = isSingleRedis - ? new Redis( - redisUri ?? { - host: redisHost, - port: redisPort, - password: redisPassword, - } - ) - : new Redis.Cluster( - [ - { - host: redisHost, - port: redisPort, - }, - ], - { - dnsLookup: (address, callback) => callback(null, address, 4), - redisOptions, + // Cache hit + this.logger.debug({ + ...LOGGER_DATA, + message: `Cache hit for form ${formId}` + }); + configObj = JSON.parse(jsonDataString); + if (!sessionValidated) { + // Validate cached form once per session + this.logger.debug({ + ...LOGGER_DATA, + message: `First access of form ${formId} in yar session ${request.yar.id}, validating cache` + }); + const isValid = await this.validateCachedForm(formId, configObj.hash); + if (!isValid) { + this.logger.info({ + ...LOGGER_DATA, + message: `Cache stale for form ${formId}, fetching fresh version` + }); + const freshConfig = await this.fetchAndCacheForm(formId); + if (freshConfig) { + configObj = freshConfig; } - ); - return client; + } else { + this.logger.debug({ + ...LOGGER_DATA, + message: `Cache valid for form ${formId}` + }); + } + } else { + this.logger.debug({ + ...LOGGER_DATA, + message: `Form ${formId} already validated in yar session ${request.yar.id}` + }); + } } else { - console.log({ + // Cache miss - fetch from Pre-Award API + this.logger.info({ ...LOGGER_DATA, - message: `[FORM-CACHE] using memory caching`, - }) + message: `Cache miss for form ${formId}, fetching from Pre-Award API` + }); + configObj = await this.fetchAndCacheForm(formId); + if (!configObj) { + throw Boom.notFound(`Form '${formId}' not found`); + } + } + if (!sessionValidated) { + // Mark form as validated in this session + this.logger.debug({ + ...LOGGER_DATA, + message: `Marking form ${formId} as validated in yar session ${request.yar.id}` + }); + await this.formStorage.setex(formSessionCacheKey, sessionTimeout / 1000, true); } + return new AdapterFormModel(configObj.configuration, { + basePath: formId, + hash: configObj.hash, + previewMode: true, + translationEn: translations.en, + translationCy: translations.cy + }); } + + private getRedisClient(): Redis | null { + const client = createRedisClient(); + if (!client) { + console.log({ + ...LOGGER_DATA, + message: `[FORM-CACHE] using memory caching`, + }); + } + return client; + } } export const catboxProvider = () => { @@ -353,49 +311,16 @@ export const catboxProvider = () => { */ const provider = { constructor: redisHost || redisUri ? CatboxRedis.Engine : CatboxMemory.Engine, - options: {}, + options: {partition}, }; if (redisHost || redisUri) { - console.log("Starting redis session management") - const redisOptions: { - password?: string; - tls?: {}; - } = {}; - - if (redisPassword) { - redisOptions.password = redisPassword; - } - - if (redisTls) { - redisOptions.tls = {}; - } - - const client = isSingleRedis - ? new Redis( - redisUri ?? { - host: redisHost, - port: redisPort, - password: redisPassword, - } - ) - : new Redis.Cluster( - [ - { - host: redisHost, - port: redisPort, - }, - ], - { - dnsLookup: (address, callback) => callback(null, address, 4), - redisOptions, - } - ); - provider.options = {client, partition}; - console.log(`Redis Url : ${redisUri} session management`); + console.log("Starting redis session management"); + const client = createRedisClient(); + provider.options = {client, partition}; + console.log(`Redis Url : ${redisUri} session management`); } else { - console.log("Starting in memory session management") - provider.options = {partition}; + console.log("Starting in memory session management"); } return provider; diff --git a/runner/src/server/services/PreAwardApiService.ts b/runner/src/server/services/PreAwardApiService.ts new file mode 100644 index 00000000..7b276ad4 --- /dev/null +++ b/runner/src/server/services/PreAwardApiService.ts @@ -0,0 +1,111 @@ +import { HapiServer } from "../types"; +import { config } from "../plugins/utils/AdapterConfigurationSchema"; +import Boom from "boom"; +import wreck from "@hapi/wreck"; + +export interface FormHashResponse { + hash: string; +} + +export interface PublishedFormResponse { + configuration: any; + hash: string; +} + +const LOGGER_DATA = { + class: "PreAwardApiService", +} + +export class PreAwardApiService { + private logger: any; + private apiBaseUrl: string; + private wreck: typeof wreck; + + constructor(server: HapiServer) { + this.logger = server.logger; + this.apiBaseUrl = config.formStoreApiHost; + this.wreck = wreck.defaults({ + timeout: 10000, + headers: { + 'accept': 'application/json', + 'content-type': 'application/json' + } + }); + this.logger.info({ + ...LOGGER_DATA, + message: `Service initialized with base URL: ${this.apiBaseUrl}` + }); + } + + /** + * Fetches the published form data including hash for a specific form. + * This is the primary method used by the cache service when a form + * is requested by a user. It only returns forms that have been + * explicitly published in the Pre-Award system. + */ + async getPublishedForm(name: string): Promise { + const url = `${this.apiBaseUrl}/${name}/published`; + this.logger.info({ + ...LOGGER_DATA, + message: `Fetching published form: ${name}`, + url: url + }); + try { + const { payload } = await this.wreck.get(url, {json: true}); + this.logger.info({ + ...LOGGER_DATA, + message: `Successfully fetched published form: ${name}` + }); + return payload as PublishedFormResponse; + } catch (error: any) { + // Handle 404 - form doesn't exist or isn't published + if (error.output?.statusCode === 404) { + this.logger.info({ + ...LOGGER_DATA, + message: `Form ${name} not found or not published in Pre-Award API` + }); + return null; + } + // Handle other errors (network, timeout, server errors) + this.logger.error({ + ...LOGGER_DATA, + message: `Failed to fetch published form ${name}`, + error: error.message + }); + // Don't expose internal error details to the client + throw Boom.serverUnavailable('Pre-Award API is temporarily unavailable'); + } + } + + /** + * Fetches just the hash of a published form. + * This lightweight endpoint allows us to validate our cache without + * downloading the entire form definition. We use this for periodic + * cache freshness checks. + */ + async getFormHash(name: string): Promise { + const url = `${this.apiBaseUrl}/${name}/hash`; + try { + const { payload } = await this.wreck.get(url, {json: true}); + const data = payload as FormHashResponse; + this.logger.debug({ + ...LOGGER_DATA, + message: `Retrieved hash for form ${name}` + }); + return data.hash; + } catch (error: any) { + if (error.output?.statusCode === 404) { + // Form doesn't exist or isn't published - this is normal + return null; + } + // For hash validation failures, we don't want to fail the entire request. + // We'll continue using the cached version and try again later. + this.logger.warn({ + ...LOGGER_DATA, + message: `Could not fetch hash for form ${name}, will use cached version`, + error: error.message + }); + return null; + } + } +} \ No newline at end of file diff --git a/runner/src/server/types.ts b/runner/src/server/types.ts index 659ff259..82309826 100644 --- a/runner/src/server/types.ts +++ b/runner/src/server/types.ts @@ -17,6 +17,7 @@ import {AdapterCacheService, S3UploadService} from "./services"; import {AdapterStatusService} from "./services"; import {WebhookService} from "./services/WebhookService"; import {TranslationLoaderService} from "./services/TranslationLoaderService"; +import { PreAwardApiService } from "./services/PreAwardApiService"; export type ChangeRequest = { @@ -32,6 +33,7 @@ export type Services = (services: string[]) => { webhookService: WebhookService; adapterStatusService: AdapterStatusService; translationLoaderService: TranslationLoaderService; + preAwardApiService: PreAwardApiService; }; export type RouteConfig = {