Skip to content

Commit d60242c

Browse files
HenryHengZJHenryHengZJ
authored
Feat/add multer s3 (#3854)
* add multer s3 * add types multer s3 * update multer s3 implementation * Revert "update multer s3 implementation" This reverts commit 9a25bf5. * update storage utils * update multer storage type on routes * revert getMulterStorage * revert getMulterStorage * update getmulterstorage * update getmulterstorage * update getmulterstorage
1 parent 15d06ec commit d60242c

File tree

19 files changed

+289
-62
lines changed

19 files changed

+289
-62
lines changed

‎packages/components/src/storageUtils.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,37 @@ export const addSingleFileToStorage = async (mime: string, bf: Buffer, fileName:
120120
}
121121
}
122122

123+
export const getFileFromUpload = async (filePath: string): Promise<Buffer> => {
124+
const storageType = getStorageType()
125+
if (storageType === 's3') {
126+
const { s3Client, Bucket } = getS3Config()
127+
128+
let Key = filePath
129+
// remove the first '/' if it exists
130+
if (Key.startsWith('/')) {
131+
Key = Key.substring(1)
132+
}
133+
const getParams = {
134+
Bucket,
135+
Key
136+
}
137+
138+
const response = await s3Client.send(new GetObjectCommand(getParams))
139+
const body = response.Body
140+
if (body instanceof Readable) {
141+
const streamToString = await body.transformToString('base64')
142+
if (streamToString) {
143+
return Buffer.from(streamToString, 'base64')
144+
}
145+
}
146+
// @ts-ignore
147+
const buffer = Buffer.concat(response.Body.toArray())
148+
return buffer
149+
} else {
150+
return fs.readFileSync(filePath)
151+
}
152+
}
153+
123154
export const getFileFromStorage = async (file: string, ...paths: string[]): Promise<Buffer> => {
124155
const storageType = getStorageType()
125156
const sanitizedFilename = _sanitizeFilename(file)
@@ -183,6 +214,20 @@ export const removeFilesFromStorage = async (...paths: string[]) => {
183214
}
184215
}
185216

217+
export const removeSpecificFileFromUpload = async (filePath: string) => {
218+
const storageType = getStorageType()
219+
if (storageType === 's3') {
220+
let Key = filePath
221+
// remove the first '/' if it exists
222+
if (Key.startsWith('/')) {
223+
Key = Key.substring(1)
224+
}
225+
await _deleteS3Folder(Key)
226+
} else {
227+
fs.unlinkSync(filePath)
228+
}
229+
}
230+
186231
export const removeSpecificFileFromStorage = async (...paths: string[]) => {
187232
const storageType = getStorageType()
188233
if (storageType === 's3') {

‎packages/server/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@
9191
"moment": "^2.29.3",
9292
"moment-timezone": "^0.5.34",
9393
"multer": "^1.4.5-lts.1",
94+
"multer-s3": "^3.0.1",
9495
"mysql2": "^3.11.3",
9596
"openai": "^4.57.3",
9697
"pg": "^8.11.1",
@@ -110,6 +111,7 @@
110111
"@types/cors": "^2.8.12",
111112
"@types/crypto-js": "^4.1.1",
112113
"@types/multer": "^1.4.7",
114+
"@types/multer-s3": "^3.0.3",
113115
"@types/sanitize-html": "^2.9.5",
114116
"concurrently": "^7.1.0",
115117
"cypress": "^13.13.0",

‎packages/server/src/controllers/openai-assistants-vector-store/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ const uploadFilesToAssistantVectorStore = async (req: Request, res: Response, ne
143143
// Address file name with special characters: https://github.com/expressjs/multer/issues/1104
144144
file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
145145
uploadFiles.push({
146-
filePath: file.path,
146+
filePath: file.path ?? file.key,
147147
fileName: file.originalname
148148
})
149149
}

‎packages/server/src/controllers/openai-assistants/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ const uploadAssistantFiles = async (req: Request, res: Response, next: NextFunct
8484
// Address file name with special characters: https://github.com/expressjs/multer/issues/1104
8585
file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
8686
uploadFiles.push({
87-
filePath: file.path,
87+
filePath: file.path ?? file.key,
8888
fileName: file.originalname
8989
})
9090
}

‎packages/server/src/index.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,20 @@ declare global {
3333
interface Request {
3434
io?: Server
3535
}
36+
namespace Multer {
37+
interface File {
38+
bucket: string
39+
key: string
40+
acl: string
41+
contentType: string
42+
contentDisposition: null
43+
storageClass: string
44+
serverSideEncryption: null
45+
metadata: any
46+
location: string
47+
etag: string
48+
}
49+
}
3650
}
3751
}
3852

‎packages/server/src/routes/attachments/index.ts‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
11
import express from 'express'
2-
import multer from 'multer'
32
import attachmentsController from '../../controllers/attachments'
4-
import { getUploadPath } from '../../utils'
3+
import { getMulterStorage } from '../../utils'
54

65
const router = express.Router()
76

8-
const upload = multer({ dest: getUploadPath() })
9-
107
// CREATE
11-
router.post('/:chatflowId/:chatId', upload.array('files'), attachmentsController.createAttachment)
8+
router.post('/:chatflowId/:chatId', getMulterStorage().array('files'), attachmentsController.createAttachment)
129

1310
export default router

‎packages/server/src/routes/documentstore/index.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
import express from 'express'
2-
import multer from 'multer'
3-
import { getUploadPath } from '../../utils'
42
import documentStoreController from '../../controllers/documentstore'
3+
import { getMulterStorage } from '../../utils'
54

65
const router = express.Router()
7-
const upload = multer({ dest: getUploadPath() })
86

9-
router.post(['/upsert/', '/upsert/:id'], upload.array('files'), documentStoreController.upsertDocStoreMiddleware)
7+
router.post(['/upsert/', '/upsert/:id'], getMulterStorage().array('files'), documentStoreController.upsertDocStoreMiddleware)
108

119
router.post(['/refresh/', '/refresh/:id'], documentStoreController.refreshDocStoreMiddleware)
1210

‎packages/server/src/routes/openai-assistants-files/index.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
import express from 'express'
2-
import multer from 'multer'
32
import openaiAssistantsController from '../../controllers/openai-assistants'
4-
import { getUploadPath } from '../../utils'
3+
import { getMulterStorage } from '../../utils'
54

65
const router = express.Router()
7-
const upload = multer({ dest: getUploadPath() })
86

97
router.post('/download/', openaiAssistantsController.getFileFromAssistant)
10-
router.post('/upload/', upload.array('files'), openaiAssistantsController.uploadAssistantFiles)
8+
router.post('/upload/', getMulterStorage().array('files'), openaiAssistantsController.uploadAssistantFiles)
119

1210
export default router

‎packages/server/src/routes/openai-assistants-vector-store/index.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
import express from 'express'
2-
import multer from 'multer'
32
import openaiAssistantsVectorStoreController from '../../controllers/openai-assistants-vector-store'
4-
import { getUploadPath } from '../../utils'
3+
import { getMulterStorage } from '../../utils'
54

65
const router = express.Router()
7-
const upload = multer({ dest: getUploadPath() })
86

97
// CREATE
108
router.post('/', openaiAssistantsVectorStoreController.createAssistantVectorStore)
@@ -22,7 +20,7 @@ router.put(['/', '/:id'], openaiAssistantsVectorStoreController.updateAssistantV
2220
router.delete(['/', '/:id'], openaiAssistantsVectorStoreController.deleteAssistantVectorStore)
2321

2422
// POST
25-
router.post('/:id', upload.array('files'), openaiAssistantsVectorStoreController.uploadFilesToAssistantVectorStore)
23+
router.post('/:id', getMulterStorage().array('files'), openaiAssistantsVectorStoreController.uploadFilesToAssistantVectorStore)
2624

2725
// DELETE
2826
router.patch(['/', '/:id'], openaiAssistantsVectorStoreController.deleteFilesFromAssistantVectorStore)

‎packages/server/src/routes/predictions/index.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
import express from 'express'
2-
import multer from 'multer'
32
import predictionsController from '../../controllers/predictions'
4-
import { getUploadPath } from '../../utils'
3+
import { getMulterStorage } from '../../utils'
54

65
const router = express.Router()
76

8-
const upload = multer({ dest: getUploadPath() })
9-
107
// CREATE
11-
router.post(['/', '/:id'], upload.array('files'), predictionsController.getRateLimiterMiddleware, predictionsController.createPrediction)
8+
router.post(
9+
['/', '/:id'],
10+
getMulterStorage().array('files'),
11+
predictionsController.getRateLimiterMiddleware,
12+
predictionsController.createPrediction
13+
)
1214

1315
export default router

0 commit comments

Comments
 (0)