Prototype optimisations (#211)

* Add memory logging and reduce storage lengths

* Update doc

Author: edwardhorsford

app.js

@@ -1,3 +1,6 @@
+// Suppress deprecation warning from dependencies using util.isArray
+process.noDeprecation = true
+
 // Core dependencies
 const {
   createReadStream,
@@ -96,7 +99,7 @@ const sessionName = `nhsuk-prototype-kit-${Buffer.from(config.serviceName, 'utf8
 const sessionOptions = {
   secret: sessionName,
   cookie: {
-    maxAge: 1000 * 60 * 60 * 4 // 4 hours
+    maxAge: 1000 * 60 * 60 * 1 // 1 hour (reduced from 4 to limit session accumulation)
   }
 }
 

app/lib/utils/regenerate-data.js

@@ -11,12 +11,19 @@ async function regenerateData(req) {
   const generatedDataPath = resolve(dataDirectory, 'generated')
   const generationInfoPath = join(generatedDataPath, 'generation-info.json')
 
+  // Also clear lib/utils.js cache so its sessionDataDefaults gets refreshed
+  // (it caches sessionDataDefaults at module load time)
+  const libUtilsPath = resolve(__dirname, '../../../lib/utils.js')
+
   // Generate new data
   await generateData()
 
   // Clear the require cache for session data defaults
   delete require.cache[require.resolve(sessionDataPath)]
 
+  // Clear lib/utils.js cache so it reloads with fresh sessionDataDefaults
+  delete require.cache[require.resolve(libUtilsPath)]
+
   // Clear cache for the generated JSON files
   Object.keys(require.cache).forEach((key) => {
     if (key.startsWith(generatedDataPath)) {

app/routes.js

@@ -10,12 +10,54 @@ const { resetCallSequence } = require('./lib/utils/random')
 
 const router = express.Router()
 
+// Memory logging - tracks growth and logs significant changes
+let requestCount = 0
+let lastLoggedRss = 0
+const logMemory = (context, sessionData) => {
+  const mem = process.memoryUsage()
+  const rss = mem.rss / 1024 / 1024
+  let sessionInfo = ''
+  if (sessionData) {
+    try {
+      const sessionStr = JSON.stringify(sessionData)
+      sessionInfo = `, session: ${(sessionStr.length / 1024 / 1024).toFixed(2)}MB`
+    } catch (e) {
+      sessionInfo = ', session: [error]'
+    }
+  }
+  console.log(
+    `[Memory ${context}] heap: ${(mem.heapUsed / 1024 / 1024).toFixed(1)}MB, rss: ${rss.toFixed(1)}MB, requests: ${requestCount}${sessionInfo}`
+  )
+  lastLoggedRss = rss
+}
+
+// Log memory on startup
+logMemory('startup')
+
 // Keep regeneration middleware before other routes
 router.use(async (req, res, next) => {
   try {
+    requestCount++
+    const isStaticRequest = req.path.match(
+      /\.(js|css|ico|png|jpg|svg|woff|woff2)$/
+    )
+
+    // Log memory if RSS has grown significantly (>20MB) since last log
+    const currentRss = process.memoryUsage().rss / 1024 / 1024
+    if (!isStaticRequest && currentRss - lastLoggedRss > 20) {
+      logMemory(`growth on ${req.method} ${req.path}`, req.session?.data)
+    }
+
+    // Also log periodically
+    if (!isStaticRequest && requestCount % 50 === 0) {
+      logMemory(`request #${requestCount}`, req.session?.data)
+    }
+
     if (needsRegeneration(req.session.data?.generationInfo)) {
+      logMemory('before-regeneration')
       console.log('Regenerating data for new day...')
       await regenerateData(req)
+      logMemory('after-regeneration')
     }
     next()
   } catch (err) {

app/routes/events.js

@@ -2819,4 +2819,4 @@ module.exports = (router) => {
     )
     res.redirect(returnUrl)
   })
-}
+}

lib/middleware/prototype-admin-routes.js

@@ -59,26 +59,35 @@ router.all('/reset-session-data', (req, res) => {
     returnPage = '/'
   }
 
-  console.log('Clearing session data')
+  // Log memory before reset
+  const memBefore = process.memoryUsage()
+  console.log(
+    `[Memory before-reset] heap: ${(memBefore.heapUsed / 1024 / 1024).toFixed(1)}MB, rss: ${(memBefore.rss / 1024 / 1024).toFixed(1)}MB`
+  )
 
   const dataDirectory = join(__dirname, '../../app/data')
   const sessionDataPath = resolve(dataDirectory, 'session-data-defaults.js')
-  const generatedDataPath = resolve(dataDirectory, 'generated')
-
-  // Clear the require cache for session data defaults
-  // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-  delete require.cache[require.resolve(sessionDataPath)]
-
-  // Clear cache for the generated JSON files
-  Object.keys(require.cache).forEach((key) => {
-    if (key.startsWith(generatedDataPath)) {
-      // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-      delete require.cache[key]
-    }
-  })
 
-  // Load fresh session defaults after clearing cache
-  req.session.data = require(sessionDataPath)
+  // Get the cached session defaults
+  // Note: We don't clear require cache here as that causes memory growth
+  // when done repeatedly (JSON parsing artifacts don't get GC'd properly).
+  // The autoStoreData middleware will merge these defaults with session data
+  // on the next request anyway.
+  const sessionDefaults = require(sessionDataPath)
+
+  // Clear existing session data and reset to defaults
+  // Using Object.assign to create a shallow copy - the arrays (participants,
+  // events, clinics) remain shared references which is fine since sessions
+  // don't typically modify the original arrays, just individual items
+  Object.keys(req.session.data).forEach((key) => delete req.session.data[key])
+  Object.assign(req.session.data, sessionDefaults)
+
+  // Log memory after reset
+  const memAfter = process.memoryUsage()
+  console.log(
+    `[Memory after-reset] heap: ${(memAfter.heapUsed / 1024 / 1024).toFixed(1)}MB, rss: ${(memAfter.rss / 1024 / 1024).toFixed(1)}MB`
+  )
+
   req.flash('success', 'Session data cleared')
 
   res.render('reset-done', {