fix: improve request cancellation to keep the s3Client queue low (#548)

Author: fenos

src/app.ts

@@ -50,6 +50,7 @@ const build = (opts: buildOpts = {}): FastifyInstance => {
   app.addSchema(schemas.authSchema)
   app.addSchema(schemas.errorSchema)
 
+  app.register(plugins.signals)
   app.register(plugins.tenantId)
   app.register(plugins.metrics({ enabledEndpoint: !isMultitenant }))
   app.register(plugins.tracing)

src/config.ts

@@ -25,6 +25,9 @@ type StorageConfigType = {
   storageS3Endpoint?: string
   storageS3ForcePathStyle?: boolean
   storageS3Region: string
+  storageS3ClientTimeout: number
+  storageS3UploadTimeout: number
+  storageS3DownloadTimeout: number
   isMultitenant: boolean
   jwtSecret: string
   jwtAlgorithm: string
@@ -274,6 +277,15 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType {
       getOptionalConfigFromEnv('STORAGE_S3_FORCE_PATH_STYLE', 'GLOBAL_S3_FORCE_PATH_STYLE') ===
       'true',
     storageS3Region: getOptionalConfigFromEnv('STORAGE_S3_REGION', 'REGION') as string,
+    storageS3ClientTimeout: Number(
+      getOptionalConfigFromEnv('STORAGE_S3_CLIENT_TIMEOUT') || `${1000 * 600}` // 10m
+    ),
+    storageS3DownloadTimeout: Number(
+      getOptionalConfigFromEnv('STORAGE_S3_DOWNLOAD_TIMEOUT') || `${1000 * 43200}` //12h
+    ),
+    storageS3UploadTimeout: Number(
+      getOptionalConfigFromEnv('STORAGE_S3_UPLOAD_TIMEOUT') || `${1000 * 1200}` // 20m
+    ),
 
     // DB - Migrations
     dbAnonRole: getOptionalConfigFromEnv('DB_ANON_ROLE') || 'anon',

src/http/plugins/index.ts

@@ -9,3 +9,4 @@ export * from './metrics'
 export * from './xml'
 export * from './signature-v4'
 export * from './tracing'
+export * from './signals'

src/http/plugins/signals.ts

@@ -0,0 +1,43 @@
+import fastifyPlugin from 'fastify-plugin'
+import { FastifyInstance } from 'fastify'
+
+declare module 'fastify' {
+  interface FastifyRequest {
+    signals: {
+      body: AbortController
+      response: AbortController
+      disconnect: AbortController
+    }
+  }
+}
+
+export const signals = fastifyPlugin(
+  async function (fastify: FastifyInstance) {
+    fastify.addHook('onRequest', async (req, res) => {
+      req.signals = {
+        body: new AbortController(),
+        response: new AbortController(),
+        disconnect: new AbortController(),
+      }
+
+      // Client terminated the request before the body was fully received
+      res.raw.once('close', () => {
+        req.signals.response.abort()
+
+        if (!req.signals.disconnect.signal.aborted) {
+          req.signals.disconnect.abort()
+        }
+      })
+    })
+
+    // Client terminated the request before the body was fully sent
+    fastify.addHook('onRequestAbort', async (req) => {
+      req.signals.body.abort()
+
+      if (!req.signals.disconnect.signal.aborted) {
+        req.signals.disconnect.abort()
+      }
+    })
+  },
+  { name: 'request-signals' }
+)

src/http/routes/object/getObject.ts

@@ -78,6 +78,7 @@ async function requestHandler(
     key: s3Key,
     version: obj.version,
     download,
+    signal: request.signals.disconnect.signal,
   })
 }
 

src/http/routes/object/getPublicObject.ts

@@ -66,6 +66,7 @@ export default async function routes(fastify: FastifyInstance) {
         key: s3Key,
         version: obj.version,
         download,
+        signal: request.signals.disconnect.signal,
       })
     }
   )

src/http/routes/object/getSignedObject.ts

@@ -92,6 +92,7 @@ export default async function routes(fastify: FastifyInstance) {
         version: obj.version,
         download,
         expires: new Date(exp * 1000).toUTCString(),
+        signal: request.signals.disconnect.signal,
       })
     }
   )

src/http/routes/render/renderAuthenticatedImage.ts

@@ -61,6 +61,7 @@ export default async function routes(fastify: FastifyInstance) {
         key: s3Key,
         version: obj.version,
         download,
+        signal: request.signals.disconnect.signal,
       })
     }
   )

src/http/routes/render/renderPublicImage.ts

@@ -66,6 +66,7 @@ export default async function routes(fastify: FastifyInstance) {
         key: s3Key,
         version: obj.version,
         download,
+        signal: request.signals.disconnect.signal,
       })
     }
   )

src/http/routes/render/renderSignedImage.ts

@@ -94,6 +94,7 @@ export default async function routes(fastify: FastifyInstance) {
           version: obj.version,
           download,
           expires: new Date(exp * 1000).toUTCString(),
+          signal: request.signals.disconnect.signal,
         })
     }
   )