feat: Add mtime as optional ETag algorithm in file backend alongside default md5 (#613)

* fix: Optimize etag generation to improve file backend performance

* feat: Add configurable etag algorithm for file backend

* md5 as default

Author: ciscorn

.env.sample

@@ -100,7 +100,7 @@ AWS_SECRET_ACCESS_KEY=secret1234
 # File Backend
 #######################################
 STORAGE_FILE_BACKEND_PATH=./data
-
+STORAGE_FILE_ETAG_ALGORITHM=md5
 
 #######################################
 # Image Transformation

src/config.ts

@@ -20,6 +20,7 @@ type StorageConfigType = {
   uploadFileSizeLimit: number
   uploadFileSizeLimitStandard?: number
   storageFilePath?: string
+  storageFileEtagAlgorithm: 'mtime' | 'md5'
   storageS3MaxSockets: number
   storageS3Bucket: string
   storageS3Endpoint?: string
@@ -273,6 +274,7 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType {
       'STORAGE_FILE_BACKEND_PATH',
       'FILE_STORAGE_BACKEND_PATH'
     ),
+    storageFileEtagAlgorithm: getOptionalConfigFromEnv('STORAGE_FILE_ETAG_ALGORITHM') || 'md5',
 
     // Storage - S3
     storageS3MaxSockets: parseInt(

src/storage/backend/file.ts

@@ -45,13 +45,25 @@ const METADATA_ATTR_KEYS = {
 export class FileBackend implements StorageBackendAdapter {
   client = null
   filePath: string
+  etagAlgorithm: 'mtime' | 'md5'
 
   constructor() {
-    const { storageFilePath } = getConfig()
+    const { storageFilePath, storageFileEtagAlgorithm } = getConfig()
     if (!storageFilePath) {
       throw new Error('FILE_STORAGE_BACKEND_PATH env variable not set')
     }
     this.filePath = storageFilePath
+    this.etagAlgorithm = storageFileEtagAlgorithm
+  }
+
+  private async etag(file: string, stats: fs.Stats): Promise<string> {
+    if (this.etagAlgorithm === 'md5') {
+      const checksum = await fileChecksum(file)
+      return `"${checksum}"`
+    } else if (this.etagAlgorithm === 'mtime') {
+      return `"${stats.mtimeMs.toString(16)}-${stats.size.toString(16)}"`
+    }
+    throw new Error('FILE_STORAGE_ETAG_ALGORITHM env variable must be either "mtime" or "md5"')
   }
 
   /**
@@ -70,7 +82,7 @@ export class FileBackend implements StorageBackendAdapter {
     // 'Range: bytes=#######-######
     const file = path.resolve(this.filePath, withOptionalVersion(`${bucketName}/${key}`, version))
     const data = await fs.stat(file)
-    const checksum = await fileChecksum(file)
+    const eTag = await this.etag(file, data)
     const fileSize = data.size
     const { cacheControl, contentType } = await this.getFileMetadata(file)
     const lastModified = new Date(0)
@@ -92,7 +104,7 @@ export class FileBackend implements StorageBackendAdapter {
           contentRange: `bytes ${startRange}-${endRange}/${fileSize}`,
           httpStatusCode: 206,
           size: size,
-          eTag: checksum,
+          eTag,
           contentLength: chunkSize,
         },
         httpStatusCode: 206,
@@ -107,7 +119,7 @@ export class FileBackend implements StorageBackendAdapter {
           lastModified: lastModified,
           httpStatusCode: 200,
           size: data.size,
-          eTag: checksum,
+          eTag,
           contentLength: fileSize,
         },
         body,
@@ -205,12 +217,12 @@ export class FileBackend implements StorageBackendAdapter {
     await this.setFileMetadata(destFile, Object.assign({}, originalMetadata, metadata))
 
     const fileStat = await fs.lstat(destFile)
-    const checksum = await fileChecksum(destFile)
+    const eTag = await this.etag(destFile, fileStat)
 
     return {
       httpStatusCode: 200,
       lastModified: fileStat.mtime,
-      eTag: checksum,
+      eTag,
     }
   }
 
@@ -252,15 +264,14 @@ export class FileBackend implements StorageBackendAdapter {
     const { cacheControl, contentType } = await this.getFileMetadata(file)
     const lastModified = new Date(0)
     lastModified.setUTCMilliseconds(data.mtimeMs)
-
-    const checksum = await fileChecksum(file)
+    const eTag = await this.etag(file, data)
 
     return {
       httpStatusCode: 200,
       size: data.size,
       cacheControl: cacheControl || 'no-cache',
       mimetype: contentType || 'application/octet-stream',
-      eTag: `"${checksum}"`,
+      eTag,
       lastModified: data.birthtime,
       contentLength: data.size,
     }