Speed up file uploading for large files

package.json

@@ -24,6 +24,7 @@
     },
     "homepage": "https://github.com/kobra-dev/dataset-api#readme",
     "dependencies": {
+        "@aws-sdk/client-s3": "^3.38.0",
         "@aws-sdk/types": "^3.15.0",
         "@types/validator": "^13.1.3",
         "aws-sdk": "^2.903.0",

reasearch.txt

@@ -0,0 +1,2 @@
+- https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property
+- 

src/app.ts

@@ -83,13 +83,14 @@ const app: FastifyPluginAsync<AppOptions> = async (
                 reply.send({ message: 'Not authorized' })
             }
         }
+
         request.user = user
     })
 
     void fastify.register(require('fastify-multipart'), {
         limits: {
             files: 1, // Maximum number of files
-            fieldSize: 2097152, // Maximum number of bytes(2MB),
+            fileSize: 1024 * 1024 * 2, // Maximum number of bytes(2MB),
         },
     })
 

src/routes/dataset.ts

@@ -1,10 +1,13 @@
 import { FastifyPluginAsync } from 'fastify'
 import { hashString } from '../utils/helpers'
+//import fs from 'fs'
+
 import {
     getFileByKey,
     deleteObject,
     doesFileExists,
     uploadFile,
+    //   createMultipart,
     updateFile,
 } from '../utils/s3'
 
@@ -19,23 +22,28 @@ const datasets: FastifyPluginAsync = async (fastify, _): Promise<void> => {
                 message: 'Not authorized',
             })
 
-        const data = await request.file()
+        const options = {
+            throwFileSizeLimit: false,
+            limits: { fileSize: 1400 },
+        }
+
+        const data = await request.file(options)
+
+        console.log(data.file)
 
         if (!data)
             reply.status(400).send({
                 message: 'No dataset uploaded',
             })
 
         const extension = data.filename.split('.').pop()
-
         const allowedExtensions = ['xls', 'csv', 'xlsx', 'xlxb']
 
         if (!allowedExtensions.includes(extension.toLowerCase())) {
             reply.status(400).send({
                 message: 'Invalid dataset file',
             })
         }
-
         const doesObjectExists = await doesFileExists(
             hashString(data.filename + '@' + request.user.uid),
         )
@@ -47,7 +55,6 @@ const datasets: FastifyPluginAsync = async (fastify, _): Promise<void> => {
             })
 
         const uploadResult = await uploadFile(data, request.user.uid)
-
         reply.status(201).send({
             message: 'file uploaded successfully',
             Key: uploadResult.key,
@@ -68,7 +75,7 @@ const datasets: FastifyPluginAsync = async (fastify, _): Promise<void> => {
         const doesObjectExists = await doesFileExists(key)
 
         if (!doesObjectExists)
-            reply.status(404).send({ message: "File doesn't exists" })
+            reply.status(404).send({ message: "File doesn't exist" })
 
         const isDeleted = await deleteObject(key)
 

src/utils/s3-fs.ts

@@ -0,0 +1,87 @@
+import {
+    S3,
+    CreateMultipartUploadCommand,
+    CreateMultipartUploadCommandInput,
+    UploadPartCommandInput,
+    UploadPartCommand,
+    CompleteMultipartUploadCommandInput,
+    CompleteMultipartUploadCommand,
+} from '@aws-sdk/client-s3'
+
+const client = new S3({ region: 'us-west-2' })
+
+const createParams: CreateMultipartUploadCommandInput = {
+    Bucket: process.env.BUCKET_NAME,
+    Key: process.env.ACCESS_KEY,
+}
+
+async function upload(file: Buffer) {
+    try {
+        const createUploadResponse = await client.send(
+            new CreateMultipartUploadCommand(createParams),
+        )
+        const { Bucket, Key } = createParams
+        const { UploadId } = createUploadResponse
+        console.log('Upload initiated. Upload ID: ', UploadId)
+
+        // 5MB is the minimum part size
+        // Last part can be any size (no min.)
+        // Single part is treated as last part (no min.)
+        const partSize = 1024 * 1024 * 5 // 5MB
+        const fileSize = file.length
+        const numParts = Math.ceil(fileSize / partSize)
+
+        const uploadedParts = []
+        let remainingBytes = fileSize
+
+        for (let i = 1; i <= numParts; i++) {
+            let startOfPart = fileSize - remainingBytes
+            let endOfPart = Math.min(partSize, startOfPart + remainingBytes)
+
+            if (i > 1) {
+                endOfPart = startOfPart + Math.min(partSize, remainingBytes)
+                startOfPart += 1
+            }
+
+            const uploadParams: UploadPartCommandInput = {
+                // add 1 to endOfPart due to slice end being non-inclusive
+                Body: file.slice(startOfPart, endOfPart + 1),
+                Bucket,
+                Key,
+                UploadId,
+                PartNumber: i,
+            }
+            const uploadPartResponse = await client.send(
+                new UploadPartCommand(uploadParams),
+            )
+            console.log(`Part #${i} uploaded. ETag: `, uploadPartResponse.ETag)
+
+            remainingBytes -= Math.min(partSize, remainingBytes)
+
+            // For each part upload, you must record the part number and the ETag value.
+            // You must include these values in the subsequent request to complete the multipart upload.
+            // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
+            uploadedParts.push({ PartNumber: i, ETag: uploadPartResponse.ETag })
+        }
+
+        const completeParams: CompleteMultipartUploadCommandInput = {
+            Bucket,
+            Key,
+            UploadId,
+            MultipartUpload: {
+                Parts: uploadedParts,
+            },
+        }
+        console.log('Completing upload...')
+        const completeData = await client.send(
+            new CompleteMultipartUploadCommand(completeParams),
+        )
+        console.log('Upload complete: ', completeData.Key, '\n---')
+    } catch (e) {
+        throw e
+    }
+}
+
+async function get_object() {}
+
+async function delete_object(key: string) {}

src/utils/s3.ts

@@ -81,3 +81,10 @@ export async function deleteObject(Key: string): Promise<boolean> {
         return false
     }
 }
+
+//export async function createMultipart(file: any, Key: string) {
+// TODO:
+// Divide a file to upload into chunks
+// Upload  chunk one by one
+// Complete file uploading
+//}

test/plugins/support.test.ts

@@ -3,9 +3,9 @@ import Fastify from 'fastify'
 import Support from '../../src/plugins/support'
 
 test('support works standalone', async (t) => {
-  const fastify = Fastify()
-  void fastify.register(Support)
-  await fastify.ready()
+    const fastify = Fastify()
+    void fastify.register(Support)
+    await fastify.ready()
 
-  t.equal(fastify.someSupport(), 'hugs')
+    t.equal(fastify.someSupport(), 'hugs')
 })