From 295e6d2da24b37cfbe7fc477370d1daaf026e952 Mon Sep 17 00:00:00 2001 From: xqvvu Date: Thu, 25 Sep 2025 19:39:28 +0800 Subject: [PATCH 1/8] feat: migrate to minio --- packages/service/common/s3/buckets/base.ts | 87 +++++++++ packages/service/common/s3/buckets/manager.ts | 24 +++ packages/service/common/s3/buckets/private.ts | 19 ++ packages/service/common/s3/buckets/public.ts | 43 +++++ packages/service/common/s3/config.ts | 10 -- packages/service/common/s3/const.ts | 20 --- packages/service/common/s3/controller.ts | 168 ------------------ packages/service/common/s3/helpers.ts | 71 ++++++++ packages/service/common/s3/index.ts | 16 -- packages/service/common/s3/interface.ts | 17 ++ packages/service/common/s3/sources/avatar.ts | 39 ++++ packages/service/common/s3/sources/base.ts | 58 ++++++ packages/service/common/s3/sources/chat.ts | 28 +++ .../common/s3/sources/dataset-image.ts | 28 +++ packages/service/common/s3/sources/dataset.ts | 28 +++ packages/service/common/s3/sources/index.ts | 24 +++ packages/service/common/s3/sources/invoice.ts | 28 +++ packages/service/common/s3/sources/rawtext.ts | 28 +++ packages/service/common/s3/type.ts | 49 ----- packages/service/common/s3/types.ts | 94 ++++++++++ packages/service/common/s3/usage-example.ts | 84 +++++++++ packages/service/package.json | 3 +- packages/web/i18n/en/account_info.json | 2 + packages/web/i18n/zh-CN/account_info.json | 2 + packages/web/i18n/zh-Hant/account_info.json | 2 + projects/app/.env.template | 5 +- projects/app/src/pages/account/info/index.tsx | 35 ++-- .../pages/api/support/user/account/update.ts | 6 +- .../api/support/user/account/updateAvatar.ts | 24 +++ .../app/src/pages/api/system/img/[...id].ts | 32 ++++ projects/app/src/pages/api/system/img/[id].ts | 21 --- projects/app/src/web/common/file/api.ts | 8 +- .../web/common/file/hooks/useUploadAvatar.tsx | 78 ++++++++ projects/app/src/web/common/file/utils.ts | 12 ++ 34 files changed, 879 insertions(+), 314 deletions(-) create mode 100644 packages/service/common/s3/buckets/base.ts create mode 100644 packages/service/common/s3/buckets/manager.ts create mode 100644 packages/service/common/s3/buckets/private.ts create mode 100644 packages/service/common/s3/buckets/public.ts delete mode 100644 packages/service/common/s3/config.ts delete mode 100644 packages/service/common/s3/const.ts delete mode 100644 packages/service/common/s3/controller.ts create mode 100644 packages/service/common/s3/helpers.ts delete mode 100644 packages/service/common/s3/index.ts create mode 100644 packages/service/common/s3/interface.ts create mode 100644 packages/service/common/s3/sources/avatar.ts create mode 100644 packages/service/common/s3/sources/base.ts create mode 100644 packages/service/common/s3/sources/chat.ts create mode 100644 packages/service/common/s3/sources/dataset-image.ts create mode 100644 packages/service/common/s3/sources/dataset.ts create mode 100644 packages/service/common/s3/sources/index.ts create mode 100644 packages/service/common/s3/sources/invoice.ts create mode 100644 packages/service/common/s3/sources/rawtext.ts delete mode 100644 packages/service/common/s3/type.ts create mode 100644 packages/service/common/s3/types.ts create mode 100644 packages/service/common/s3/usage-example.ts create mode 100644 projects/app/src/pages/api/support/user/account/updateAvatar.ts create mode 100644 projects/app/src/pages/api/system/img/[...id].ts delete mode 100644 projects/app/src/pages/api/system/img/[id].ts create mode 100644 projects/app/src/web/common/file/hooks/useUploadAvatar.tsx diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts new file mode 100644 index 000000000000..e2b269aeea03 --- /dev/null +++ b/packages/service/common/s3/buckets/base.ts @@ -0,0 +1,87 @@ +import { Client } from 'minio'; +import { + defaultS3Options, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3BucketName, + type S3Options +} from '../types'; +import type { IBucketBasicOperations } from '../interface'; +import { createObjectKey, createPresignedUrlExpires, inferContentType } from '../helpers'; + +export class S3BaseBucket implements IBucketBasicOperations { + public client: Client; + + /** + * + * @param _bucket the bucket you want to operate + * @param options the options for the s3 client + * @param afterInit the function to be called after instantiating the s3 service + */ + constructor( + private readonly _bucket: S3BucketName, + private readonly afterInit?: () => Promise | void, + public options: Partial = defaultS3Options + ) { + options = { ...defaultS3Options, ...options }; + this.options = options as S3Options; + this.client = new Client(options as S3Options); + + const init = async () => { + if (!(await this.exist())) { + await this.client.makeBucket(this._bucket); + } + await this.afterInit?.(); + }; + init(); + } + + async exist(): Promise { + return await this.client.bucketExists(this._bucket); + } + + get name(): string { + return this._bucket; + } + + upload(): Promise { + throw new Error('Method not implemented.'); + } + + download(): Promise { + throw new Error('Method not implemented.'); + } + + delete(objectKey: string): Promise { + return this.client.removeObject(this._bucket, objectKey); + } + + get(): Promise { + throw new Error('Method not implemented.'); + } + + async createPostPresignedUrl( + params: CreatePostPresignedUrlParams + ): Promise { + const maxFileSize = this.options.maxFileSize as number; + const contentType = inferContentType(params.filename); + + const policy = this.client.newPostPolicy(); + policy.setBucket(this._bucket); + policy.setContentType(contentType); + policy.setKey(createObjectKey(params)); + policy.setContentLengthRange(1, maxFileSize); + policy.setExpires(createPresignedUrlExpires(10)); + policy.setUserMetaData({ + filename: encodeURIComponent(params.filename), + visibility: params.visibility + }); + + const { formData, postURL } = await this.client.presignedPostPolicy(policy); + + return { + url: postURL, + fields: formData + }; + } +} diff --git a/packages/service/common/s3/buckets/manager.ts b/packages/service/common/s3/buckets/manager.ts new file mode 100644 index 000000000000..b3f40d0a6c69 --- /dev/null +++ b/packages/service/common/s3/buckets/manager.ts @@ -0,0 +1,24 @@ +import { type S3BaseBucket } from './base'; +import { type S3Options } from '../types'; +import { S3PublicBucket } from './public'; +import { S3PrivateBucket } from './private'; + +export class S3BucketManager { + private static instance: S3BucketManager; + private publicBucket: S3PublicBucket | null = null; + private privateBucket: S3PrivateBucket | null = null; + + private constructor() {} + + static getInstance(): S3BucketManager { + return (this.instance ??= new S3BucketManager()); + } + + getPublicBucket(options?: Partial): S3PublicBucket { + return (this.publicBucket ??= new S3PublicBucket(options)); + } + + getPrivateBucket(options?: Partial): S3PrivateBucket { + return (this.privateBucket ??= new S3PrivateBucket(options)); + } +} diff --git a/packages/service/common/s3/buckets/private.ts b/packages/service/common/s3/buckets/private.ts new file mode 100644 index 000000000000..380ac1503b50 --- /dev/null +++ b/packages/service/common/s3/buckets/private.ts @@ -0,0 +1,19 @@ +import { S3BaseBucket } from './base'; +import { + S3Buckets, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; + +export class S3PrivateBucket extends S3BaseBucket { + constructor(options?: Partial) { + super(S3Buckets.private, undefined, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return super.createPostPresignedUrl({ ...params, visibility: 'private' }); + } +} diff --git a/packages/service/common/s3/buckets/public.ts b/packages/service/common/s3/buckets/public.ts new file mode 100644 index 000000000000..bd592abf9fe1 --- /dev/null +++ b/packages/service/common/s3/buckets/public.ts @@ -0,0 +1,43 @@ +import { S3BaseBucket } from './base'; +import { createBucketPolicy } from '../helpers'; +import { + S3Buckets, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { IPublicBucketOperations } from '../interface'; + +export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperations { + constructor(options?: Partial) { + super( + S3Buckets.public, + async () => { + const bucket = this.name; + const policy = createBucketPolicy(bucket); + try { + await this.client.setBucketPolicy(bucket, policy); + } catch (error) { + // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error, + // maybe we can ignore the error, or we have other plan to handle this. + } + }, + options + ); + } + + createPublicUrl(objectKey: string): string { + const protocol = this.options.useSSL ? 'https' : 'http'; + const hostname = this.options.endPoint; + const port = this.options.port; + const bucket = this.name; + + return `${protocol}://${hostname}:${port}/${bucket}/${objectKey}`; + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return super.createPostPresignedUrl({ ...params, visibility: 'public' }); + } +} diff --git a/packages/service/common/s3/config.ts b/packages/service/common/s3/config.ts deleted file mode 100644 index 501d61be4b1a..000000000000 --- a/packages/service/common/s3/config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { S3ServiceConfig } from './type'; - -export const defualtS3Config: Omit = { - endPoint: process.env.S3_ENDPOINT || 'localhost', - port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, - useSSL: process.env.S3_USE_SSL === 'true', - accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', - secretKey: process.env.S3_SECRET_KEY || 'minioadmin', - externalBaseURL: process.env.S3_EXTERNAL_BASE_URL -}; diff --git a/packages/service/common/s3/const.ts b/packages/service/common/s3/const.ts deleted file mode 100644 index c91043dc1f96..000000000000 --- a/packages/service/common/s3/const.ts +++ /dev/null @@ -1,20 +0,0 @@ -export const mimeMap: Record = { - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.png': 'image/png', - '.gif': 'image/gif', - '.webp': 'image/webp', - '.svg': 'image/svg+xml', - '.pdf': 'application/pdf', - '.txt': 'text/plain', - '.json': 'application/json', - '.csv': 'text/csv', - '.zip': 'application/zip', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - '.doc': 'application/msword', - '.xls': 'application/vnd.ms-excel', - '.ppt': 'application/vnd.ms-powerpoint', - '.js': 'application/javascript' -}; diff --git a/packages/service/common/s3/controller.ts b/packages/service/common/s3/controller.ts deleted file mode 100644 index faaae7cdc91e..000000000000 --- a/packages/service/common/s3/controller.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { Client } from 'minio'; -import { - type FileMetadataType, - type PresignedUrlInput as UploadPresignedURLProps, - type UploadPresignedURLResponse, - type S3ServiceConfig -} from './type'; -import { defualtS3Config } from './config'; -import { randomBytes } from 'crypto'; -import { HttpProxyAgent } from 'http-proxy-agent'; -import { HttpsProxyAgent } from 'https-proxy-agent'; -import { extname } from 'path'; -import { addLog } from '../../common/system/log'; -import { getErrText } from '@fastgpt/global/common/error/utils'; -import { mimeMap } from './const'; - -export class S3Service { - private client: Client; - private config: S3ServiceConfig; - private initialized: boolean = false; - initFunction?: () => Promise; - - constructor(config?: Partial) { - this.config = { ...defualtS3Config, ...config } as S3ServiceConfig; - - this.client = new Client({ - endPoint: this.config.endPoint, - port: this.config.port, - useSSL: this.config.useSSL, - accessKey: this.config.accessKey, - secretKey: this.config.secretKey, - transportAgent: process.env.HTTP_PROXY - ? new HttpProxyAgent(process.env.HTTP_PROXY) - : process.env.HTTPS_PROXY - ? new HttpsProxyAgent(process.env.HTTPS_PROXY) - : undefined - }); - - this.initFunction = config?.initFunction; - } - - public async init() { - if (!this.initialized) { - if (!(await this.client.bucketExists(this.config.bucket))) { - addLog.debug(`Creating bucket: ${this.config.bucket}`); - await this.client.makeBucket(this.config.bucket); - } - - await this.initFunction?.(); - this.initialized = true; - } - } - - private generateFileId(): string { - return randomBytes(16).toString('hex'); - } - - private generateAccessUrl(filename: string): string { - const protocol = this.config.useSSL ? 'https' : 'http'; - const port = - this.config.port && this.config.port !== (this.config.useSSL ? 443 : 80) - ? `:${this.config.port}` - : ''; - - const externalBaseURL = this.config.externalBaseURL; - return externalBaseURL - ? `${externalBaseURL}/${this.config.bucket}/${encodeURIComponent(filename)}` - : `${protocol}://${this.config.endPoint}${port}/${this.config.bucket}/${encodeURIComponent(filename)}`; - } - - uploadFile = async (fileBuffer: Buffer, originalFilename: string): Promise => { - await this.init(); - const inferContentType = (filename: string) => { - const ext = extname(filename).toLowerCase(); - return mimeMap[ext] || 'application/octet-stream'; - }; - - if (this.config.maxFileSize && fileBuffer.length > this.config.maxFileSize) { - return Promise.reject( - `File size ${fileBuffer.length} exceeds limit ${this.config.maxFileSize}` - ); - } - - const fileId = this.generateFileId(); - const objectName = `${fileId}-${originalFilename}`; - const uploadTime = new Date(); - - const contentType = inferContentType(originalFilename); - await this.client.putObject(this.config.bucket, objectName, fileBuffer, fileBuffer.length, { - 'Content-Type': contentType, - 'Content-Disposition': `attachment; filename="${encodeURIComponent(originalFilename)}"`, - 'x-amz-meta-original-filename': encodeURIComponent(originalFilename), - 'x-amz-meta-upload-time': uploadTime.toISOString() - }); - - const metadata: FileMetadataType = { - fileId, - originalFilename, - contentType, - size: fileBuffer.length, - uploadTime, - accessUrl: this.generateAccessUrl(objectName) - }; - - return metadata; - }; - - generateUploadPresignedURL = async ({ - filepath, - contentType, - metadata, - filename - }: UploadPresignedURLProps): Promise => { - await this.init(); - const objectName = `${filepath}/${filename}`; - - try { - const policy = this.client.newPostPolicy(); - - policy.setBucket(this.config.bucket); - policy.setKey(objectName); - if (contentType) { - policy.setContentType(contentType); - } - if (this.config.maxFileSize) { - policy.setContentLengthRange(1, this.config.maxFileSize); - } - policy.setExpires(new Date(Date.now() + 10 * 60 * 1000)); // 10 mins - - policy.setUserMetaData({ - 'original-filename': encodeURIComponent(filename), - 'upload-time': new Date().toISOString(), - ...metadata - }); - - const { postURL, formData } = await this.client.presignedPostPolicy(policy); - - const response: UploadPresignedURLResponse = { - objectName, - uploadUrl: postURL, - formData - }; - - return response; - } catch (error) { - addLog.error('Failed to generate Upload Presigned URL', error); - return Promise.reject(`Failed to generate Upload Presigned URL: ${getErrText(error)}`); - } - }; - - generateDownloadUrl = (objectName: string): string => { - const pathParts = objectName.split('/'); - const encodedParts = pathParts.map((part) => encodeURIComponent(part)); - const encodedObjectName = encodedParts.join('/'); - return `${this.config.bucket}/${encodedObjectName}`; - }; - - getFile = async (objectName: string): Promise => { - const stat = await this.client.statObject(this.config.bucket, objectName); - - if (stat.size > 0) { - const accessUrl = this.generateDownloadUrl(objectName); - return accessUrl; - } - - return Promise.reject(`File ${objectName} not found`); - }; -} diff --git a/packages/service/common/s3/helpers.ts b/packages/service/common/s3/helpers.ts new file mode 100644 index 000000000000..c8b5c30f3d98 --- /dev/null +++ b/packages/service/common/s3/helpers.ts @@ -0,0 +1,71 @@ +import path from 'node:path'; +import crypto from 'node:crypto'; +import { type ContentType, type CreateObjectKeyParams, type ExtensionType, Mimes } from './types'; +import dayjs from 'dayjs'; + +/** + * + * @param filename + * @returns the Content-Type relative to the mime type + */ +export const inferContentType = (filename: string): ContentType | 'application/octet-stream' => { + const ext = path.extname(filename).toLowerCase() as ExtensionType; + return Mimes[ext] ?? 'application/octet-stream'; +}; + +/** + * Generate a date that is `minutes` minutes from now + * + * @param minutes + * @returns the date object + */ +export const createPresignedUrlExpires = (minutes: number): Date => { + return new Date(Date.now() + minutes * 60 * 1_000); +}; + +/** + * use public policy or just a custom policy + * + * @default policy public policy + * @param bucket bucket name + * @returns the policy string + */ +export const createBucketPolicy = ( + bucket: string, + policy?: 'public' | Record +): string => { + if (typeof policy === 'string' && policy !== 'public') { + throw new Error("'policy' only can be assigned to 'public' if typeof 'policy' is string"); + } + + switch (typeof policy) { + case 'string': + case 'undefined': + default: + return JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: '*', + Action: 's3:GetObject', + Resource: `arn:aws:s3:::${bucket}/*` + } + ] + }); + case 'object': + if (policy === null) { + throw new Error("Bucket policy can't be null"); + } + return JSON.stringify(policy); + } +}; + +/** + * create s3 object key by source, team ID and filename + */ +export function createObjectKey({ source, teamId, filename }: CreateObjectKeyParams): string { + const date = dayjs().format('YYYY_MM_DD'); + const id = crypto.randomBytes(16).toString('hex'); + return `${source}/${teamId}/${date}/${id}_${filename}`; +} diff --git a/packages/service/common/s3/index.ts b/packages/service/common/s3/index.ts deleted file mode 100644 index 761bd564097e..000000000000 --- a/packages/service/common/s3/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { S3Service } from './controller'; - -export const PluginS3Service = (() => { - if (!global.pluginS3Service) { - global.pluginS3Service = new S3Service({ - bucket: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', - maxFileSize: 50 * 1024 * 1024 // 50MB - }); - } - - return global.pluginS3Service; -})(); - -declare global { - var pluginS3Service: S3Service; -} diff --git a/packages/service/common/s3/interface.ts b/packages/service/common/s3/interface.ts new file mode 100644 index 000000000000..79517eaedfb1 --- /dev/null +++ b/packages/service/common/s3/interface.ts @@ -0,0 +1,17 @@ +import type { CreateObjectKeyParams } from './types'; + +export interface IBucketBasicOperations { + get name(): string; + exist(): Promise; + upload(): Promise; + download(): Promise; + delete(objectKey: string): Promise; + get(): Promise; + createPostPresignedUrl( + params: CreateObjectKeyParams + ): Promise<{ url: string; fields: Record }>; +} + +export interface IPublicBucketOperations { + createPublicUrl(objectKey: string): string; +} diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts new file mode 100644 index 000000000000..97f7666a012c --- /dev/null +++ b/packages/service/common/s3/sources/avatar.ts @@ -0,0 +1,39 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PublicBucket } from '../buckets/public'; + +class S3AvatarSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.avatar, true, options); + } + + static getInstance(options?: Partial): S3AvatarSource { + return S3BaseSource._getInstance(S3AvatarSource, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ + ...params, + source: S3Sources.avatar + }); + } + + createPublicUrl(objectKey: string): string { + return this.bucket.createPublicUrl(objectKey); + } + + removeAvatar(objectKey: string): Promise { + return this.bucket.delete(objectKey); + } +} + +export function getS3AvatarSource() { + return S3AvatarSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/base.ts b/packages/service/common/s3/sources/base.ts new file mode 100644 index 000000000000..a34dd20ffece --- /dev/null +++ b/packages/service/common/s3/sources/base.ts @@ -0,0 +1,58 @@ +import { S3BucketManager } from '../buckets/manager'; +import type { S3PrivateBucket } from '../buckets/private'; +import type { S3PublicBucket } from '../buckets/public'; +import type { + CreatePostPresignedUrlParams, + CreatePostPresignedUrlResult, + S3Options, + S3SourceType +} from '../types'; + +type Bucket = S3PublicBucket | S3PrivateBucket; + +export abstract class S3BaseSource { + protected bucket: T; + protected static instances: Map = new Map(); + + constructor( + protected readonly source: S3SourceType, + protected readonly pub: boolean = false, + options?: Partial + ) { + const manager = S3BucketManager.getInstance(); + switch (pub) { + case true: + this.bucket = manager.getPublicBucket(options) as T; + break; + case false: + this.bucket = manager.getPrivateBucket(options) as T; + } + } + + abstract createPostPresignedUrl( + params: Omit + ): Promise; + + protected static _getInstance( + constructor: new (options?: Partial) => T, + options?: Partial + ): T { + const className = constructor.name; + if (!S3BaseSource.instances.has(className)) { + S3BaseSource.instances.set(className, new constructor(options)); + } + return S3BaseSource.instances.get(className) as T; + } + + protected getBucket(): T { + return this.bucket; + } + + get bucketName(): string { + return this.bucket.name; + } + + isBucketExist(): Promise { + return this.bucket.exist(); + } +} diff --git a/packages/service/common/s3/sources/chat.ts b/packages/service/common/s3/sources/chat.ts new file mode 100644 index 000000000000..13094173d23e --- /dev/null +++ b/packages/service/common/s3/sources/chat.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3ChatSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.chat, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.chat }); + } + + static getInstance(options?: Partial): S3ChatSource { + return S3BaseSource._getInstance(S3ChatSource, options); + } +} + +export function getS3ChatSource() { + return S3ChatSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/dataset-image.ts b/packages/service/common/s3/sources/dataset-image.ts new file mode 100644 index 000000000000..11c6951b589d --- /dev/null +++ b/packages/service/common/s3/sources/dataset-image.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3DatasetImageSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources['dataset-image'], false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources['dataset-image'] }); + } + + static getInstance(options?: Partial): S3DatasetImageSource { + return S3BaseSource._getInstance(S3DatasetImageSource, options); + } +} + +export function getS3DatasetImageSource() { + return S3DatasetImageSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/dataset.ts b/packages/service/common/s3/sources/dataset.ts new file mode 100644 index 000000000000..58aa507823c3 --- /dev/null +++ b/packages/service/common/s3/sources/dataset.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3DatasetSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.dataset, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.dataset }); + } + + static getInstance(options?: Partial): S3DatasetSource { + return S3BaseSource._getInstance(S3DatasetSource, options); + } +} + +export function getS3DatasetSource() { + return S3DatasetSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/index.ts b/packages/service/common/s3/sources/index.ts new file mode 100644 index 000000000000..a19622639276 --- /dev/null +++ b/packages/service/common/s3/sources/index.ts @@ -0,0 +1,24 @@ +import { getS3AvatarSource } from './avatar'; +import { getS3ChatSource } from './chat'; +import { getS3DatasetSource } from './dataset'; +import { getS3DatasetImageSource } from './dataset-image'; +import { getS3InvoiceSource } from './invoice'; +import { getS3RawtextSource } from './rawtext'; + +export function registerSources() { + getS3AvatarSource(); + getS3ChatSource(); + getS3DatasetImageSource(); + getS3DatasetSource(); + getS3InvoiceSource(); + getS3RawtextSource(); +} + +export { + getS3AvatarSource, + getS3ChatSource, + getS3DatasetImageSource, + getS3DatasetSource, + getS3InvoiceSource, + getS3RawtextSource +}; diff --git a/packages/service/common/s3/sources/invoice.ts b/packages/service/common/s3/sources/invoice.ts new file mode 100644 index 000000000000..453020a122a5 --- /dev/null +++ b/packages/service/common/s3/sources/invoice.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3InvoiceSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.invoice, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.invoice }); + } + + static getInstance(options?: Partial): S3InvoiceSource { + return S3BaseSource._getInstance(S3InvoiceSource, options); + } +} + +export function getS3InvoiceSource() { + return S3InvoiceSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/rawtext.ts b/packages/service/common/s3/sources/rawtext.ts new file mode 100644 index 000000000000..d96792003b34 --- /dev/null +++ b/packages/service/common/s3/sources/rawtext.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3RawtextSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.rawtext, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.rawtext }); + } + + static getInstance(options?: Partial): S3RawtextSource { + return S3BaseSource._getInstance(S3RawtextSource, options); + } +} + +export function getS3RawtextSource() { + return S3RawtextSource.getInstance(); +} diff --git a/packages/service/common/s3/type.ts b/packages/service/common/s3/type.ts deleted file mode 100644 index a480530493e8..000000000000 --- a/packages/service/common/s3/type.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { ClientOptions } from 'minio'; - -export type S3ServiceConfig = { - bucket: string; - externalBaseURL?: string; - /** - * Unit: Byte - */ - maxFileSize?: number; - /** - * for executing some init function for the s3 service - */ - initFunction?: () => Promise; -} & ClientOptions; - -export type FileMetadataType = { - fileId: string; - originalFilename: string; - contentType: string; - size: number; - uploadTime: Date; - accessUrl: string; -}; - -export type PresignedUrlInput = { - filepath: string; - filename: string; - contentType?: string; - metadata?: Record; -}; - -export type UploadPresignedURLResponse = { - objectName: string; - uploadUrl: string; - formData: Record; -}; - -export type FileUploadInput = { - buffer: Buffer; - filename: string; -}; - -export enum PluginTypeEnum { - tool = 'tool' -} - -export const PluginFilePath = { - [PluginTypeEnum.tool]: 'plugin/tools' -}; diff --git a/packages/service/common/s3/types.ts b/packages/service/common/s3/types.ts new file mode 100644 index 000000000000..b7e66f74bff2 --- /dev/null +++ b/packages/service/common/s3/types.ts @@ -0,0 +1,94 @@ +import type { ClientOptions } from 'minio'; +import { HttpProxyAgent } from 'http-proxy-agent'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import { z } from 'zod'; + +export const S3MetadataSchema = z.object({ + filename: z.string(), + uploadedAt: z.date(), + accessUrl: z.string(), + contentType: z.string(), + id: z.string().length(32), + size: z.number().positive() +}); +export type S3Metadata = z.infer; + +export const Mimes = { + '.gif': 'image/gif', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + + '.csv': 'text/csv', + '.txt': 'text/plain', + + '.pdf': 'application/pdf', + '.zip': 'application/zip', + '.json': 'application/json', + '.doc': 'application/msword', + '.js': 'application/javascript', + '.xls': 'application/vnd.ms-excel', + '.ppt': 'application/vnd.ms-powerpoint', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation' +} as const; +export type ContentType = (typeof Mimes)[keyof typeof Mimes]; +export type ExtensionType = keyof typeof Mimes; + +export const defaultS3Options: { externalBaseURL?: string; maxFileSize?: number } & ClientOptions = + { + maxFileSize: 1024 ** 3, // 1GB + + useSSL: process.env.S3_USE_SSL === 'true', + endPoint: process.env.S3_ENDPOINT || 'localhost', + externalBaseURL: process.env.S3_EXTERNAL_BASE_URL, + accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', + secretKey: process.env.S3_SECRET_KEY || 'minioadmin', + port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, + transportAgent: process.env.HTTP_PROXY + ? new HttpProxyAgent(process.env.HTTP_PROXY) + : process.env.HTTPS_PROXY + ? new HttpsProxyAgent(process.env.HTTPS_PROXY) + : undefined + }; +export type S3Options = typeof defaultS3Options; + +export const S3Buckets = { + plugin: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', + public: process.env.S3_PUBLIC_BUCKET || 'fastgpt-public', + private: process.env.S3_PRIVATE_BUCKET || 'fastgpt-private' +} as const; +export type S3BucketName = (typeof S3Buckets)[keyof typeof S3Buckets]; + +export const S3SourcesSchema = z.enum([ + 'avatar', + 'chat', + 'dataset', + 'dataset-image', + 'invoice', + 'rawtext' +]); +export const S3Sources = S3SourcesSchema.enum; +export type S3SourceType = z.infer; + +export const CreateObjectKeyParamsSchema = z.object({ + filename: z.string().min(1), + source: S3SourcesSchema, + teamId: z.string().length(16) +}); +export type CreateObjectKeyParams = z.infer; + +export const CreatePostPresignedUrlParamsSchema = z.object({ + ...CreateObjectKeyParamsSchema.shape, + visibility: z.enum(['public', 'private']).default('private') +}); +export type CreatePostPresignedUrlParams = z.infer; + +export const CreatePostPresignedUrlResultSchema = z.object({ + url: z.string().min(1), + fields: z.record(z.string(), z.string()) +}); +export type CreatePostPresignedUrlResult = z.infer; diff --git a/packages/service/common/s3/usage-example.ts b/packages/service/common/s3/usage-example.ts new file mode 100644 index 000000000000..3addcb929090 --- /dev/null +++ b/packages/service/common/s3/usage-example.ts @@ -0,0 +1,84 @@ +/** + * S3 Sources 使用示例 + * 展示如何使用改进后的 S3 架构 + */ + +import { getS3AvatarSource } from './sources/avatar'; +import { getS3ChatSource } from './sources/chat'; + +// 使用示例 +export const s3UsageExample = async () => { + // 获取 avatar source（public bucket） + const avatarSource = getS3AvatarSource(); + + // 获取 chat source（private bucket） + const chatSource = getS3ChatSource(); + + // ✅ 现在可以直接调用 createPublicUrl 方法！ + const publicUrl = avatarSource.createPublicUrl('avatar/team123/2024_01_01/abc123_avatar.png'); + console.log('Avatar public URL:', publicUrl); + + // ✅ 也可以通过 getPublicBucket 获取 bucket 实例 + const publicBucket = avatarSource.getPublicBucket(); + const anotherPublicUrl = publicBucket.createPublicUrl( + 'avatar/team456/2024_01_01/def456_avatar.png' + ); + console.log('Another avatar public URL:', anotherPublicUrl); + + // ✅ 创建预签名 URL + const avatarPresignedUrl = await avatarSource.createPostPresignedUrl({ + filename: 'new-avatar.png', + teamId: '1234567890123456' + }); + console.log('Avatar presigned URL:', avatarPresignedUrl); + + // ✅ Private bucket 示例 + const chatPresignedUrl = await chatSource.createPostPresignedUrl({ + filename: 'chat-history.txt', + teamId: '1234567890123456' + }); + console.log('Chat presigned URL:', chatPresignedUrl); + + // ✅ 获取 private bucket 实例 + const privateBucket = chatSource.getPrivateBucket(); + console.log('Private bucket name:', privateBucket.name); +}; + +// 类型安全示例 +export const typeSafetyExample = () => { + const avatarSource = getS3AvatarSource(); + const chatSource = getS3ChatSource(); + + // ✅ TypeScript 现在知道 avatarSource.bucket 是 S3PublicBucket 类型 + // 所以可以安全地调用 createPublicUrl + const url1 = avatarSource.createPublicUrl('test.png'); + + // ✅ 也可以直接访问 bucket 的方法 + const url2 = avatarSource.bucket.createPublicUrl('test2.png'); + + // ❌ chatSource 没有 createPublicUrl 方法，因为它是 private bucket + // chatSource.createPublicUrl('test.png'); // TypeScript 错误 + + // ✅ 但 chatSource 可以访问基础的 bucket 方法 + console.log('Chat bucket name:', chatSource.bucketName); +}; + +// 单例验证示例 +export const singletonExample = () => { + const avatar1 = getS3AvatarSource(); + const avatar2 = getS3AvatarSource(); + + // 验证单例模式 + console.log('Same instance:', avatar1 === avatar2); // true + console.log('Same bucket:', avatar1.bucket === avatar2.bucket); // true + + const chat1 = getS3ChatSource(); + const chat2 = getS3ChatSource(); + + // 验证单例模式 + console.log('Same chat instance:', chat1 === chat2); // true + console.log('Same chat bucket:', chat1.bucket === chat2.bucket); // true + + // 不同类型的 source 使用不同的 bucket + console.log('Different buckets:', avatar1.bucket !== chat1.bucket); // true +}; diff --git a/packages/service/package.json b/packages/service/package.json index eb367ff8ed38..41438d10f7af 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -54,7 +54,8 @@ "tiktoken": "1.0.17", "tunnel": "^0.0.6", "turndown": "^7.1.2", - "winston": "^3.17.0" + "winston": "^3.17.0", + "zod": "^3.24.2" }, "devDependencies": { "@types/cookie": "^0.5.2", diff --git a/packages/web/i18n/en/account_info.json b/packages/web/i18n/en/account_info.json index 15a520732bef..c420e057d285 100644 --- a/packages/web/i18n/en/account_info.json +++ b/packages/web/i18n/en/account_info.json @@ -9,6 +9,8 @@ "app_amount": "App amount", "avatar": "Avatar", "avatar_selection_exception": "Abnormal avatar selection", + "avatar_can_only_select_one": "Avatar can only select one picture", + "avatar_can_only_select_jpg_png": "Avatar can only select jpg or png format", "balance": "balance", "billing_standard": "Standards", "cancel": "Cancel", diff --git a/packages/web/i18n/zh-CN/account_info.json b/packages/web/i18n/zh-CN/account_info.json index cd20076cdcc6..7822d8765995 100644 --- a/packages/web/i18n/zh-CN/account_info.json +++ b/packages/web/i18n/zh-CN/account_info.json @@ -9,6 +9,8 @@ "app_amount": "应用数量", "avatar": "头像", "avatar_selection_exception": "头像选择异常", + "avatar_can_only_select_one": "头像只能选择一张图片", + "avatar_can_only_select_jpg_png": "头像只能选择 jpg 或 png 格式", "balance": "余额", "billing_standard": "计费标准", "cancel": "取消", diff --git a/packages/web/i18n/zh-Hant/account_info.json b/packages/web/i18n/zh-Hant/account_info.json index 9c5560d48d91..5646df649802 100644 --- a/packages/web/i18n/zh-Hant/account_info.json +++ b/packages/web/i18n/zh-Hant/account_info.json @@ -9,6 +9,8 @@ "app_amount": "應用數量", "avatar": "頭像", "avatar_selection_exception": "頭像選擇異常", + "avatar_can_only_select_one": "頭像只能選擇一張圖片", + "avatar_can_only_select_jpg_png": "頭像只能選擇 jpg 或 png 格式", "balance": "餘額", "billing_standard": "計費標準", "cancel": "取消", diff --git a/projects/app/.env.template b/projects/app/.env.template index fd51a63b3852..eb383699b3cf 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -41,7 +41,10 @@ S3_PORT=9000 S3_USE_SSL=false S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin -S3_PLUGIN_BUCKET=fastgpt-plugin # 插件文件存储bucket +S3_PLUGIN_BUCKET=fastgpt-plugins # 插件文件存储bucket +S3_PUBLIC_BUCKET=fastgpt-public # 插件文件存储公开桶 +S3_PRIVATE_BUCKET=fastgpt-private # 插件文件存储公开桶 + # Redis URL REDIS_URL=redis://default:mypassword@127.0.0.1:6379 # mongo 数据库连接参数，本地开发连接远程数据库时，可能需要增加 directConnection=true 参数，才能连接上。 diff --git a/projects/app/src/pages/account/info/index.tsx b/projects/app/src/pages/account/info/index.tsx index 783cdfa85290..c08dab4d5bfd 100644 --- a/projects/app/src/pages/account/info/index.tsx +++ b/projects/app/src/pages/account/info/index.tsx @@ -46,6 +46,7 @@ import { getWorkorderURL } from '@/web/common/workorder/api'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import { useMount } from 'ahooks'; import MyDivider from '@fastgpt/web/components/common/MyDivider'; +import { useUploadAvatar } from '@/web/common/file/hooks/useUploadAvatar'; const RedeemCouponModal = dynamic(() => import('@/pageComponents/account/info/RedeemCouponModal'), { ssr: false @@ -140,14 +141,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { onClose: onCloseUpdateContact, onOpen: onOpenUpdateContact } = useDisclosure(); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false - }); const onclickSave = useCallback( async (data: UserType) => { @@ -164,6 +157,11 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { [reset, t, toast, updateUserInfo] ); + const { UploadAvatar, handleOpenSelectFile } = useUploadAvatar((avatar) => { + if (!userInfo) return; + onclickSave({ ...userInfo, avatar }); + }); + const labelStyles: BoxProps = { flex: '0 0 80px', color: 'var(--light-general-on-surface-lowest, var(--Gray-Modern-500, #667085))', @@ -241,6 +239,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} + {isPc ? ( {t('account_info:avatar')}  @@ -253,7 +252,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { border={theme.borders.base} overflow={'hidden'} boxShadow={'0 0 5px rgba(0,0,0,0.1)'} - onClick={onOpenSelectFile} + onClick={handleOpenSelectFile} > @@ -264,7 +263,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { flexDirection={'column'} alignItems={'center'} cursor={'pointer'} - onClick={onOpenSelectFile} + onClick={handleOpenSelectFile} > void }) => { )} + {feConfigs?.isPlus && ( {t('account_info:member_name')}  @@ -334,21 +334,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} {isOpenUpdatePsw && } {isOpenUpdateContact && } - - onSelectImage(e, { - maxW: 300, - maxH: 300, - callback: (src) => { - if (!userInfo) return; - onclickSave({ - ...userInfo, - avatar: src - }); - } - }) - } - /> ); }; diff --git a/projects/app/src/pages/api/support/user/account/update.ts b/projects/app/src/pages/api/support/user/account/update.ts index dfa451e29028..894772ed65bb 100644 --- a/projects/app/src/pages/api/support/user/account/update.ts +++ b/projects/app/src/pages/api/support/user/account/update.ts @@ -8,6 +8,7 @@ import { NextAPI } from '@/service/middleware/entry'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller'; import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; export type UserAccountUpdateQuery = {}; export type UserAccountUpdateBody = UserUpdateParams; @@ -37,6 +38,10 @@ async function handler( } // if avatar, update team member avatar if (avatar) { + if (tmb?.avatar) { + const objectKey = tmb.avatar.split('/').slice(4).join('/'); + await getS3AvatarSource().removeAvatar(objectKey); + } await MongoTeamMember.updateOne( { _id: tmbId @@ -45,7 +50,6 @@ async function handler( avatar } ).session(session); - await refreshSourceAvatar(avatar, tmb?.avatar, session); } }); diff --git a/projects/app/src/pages/api/support/user/account/updateAvatar.ts b/projects/app/src/pages/api/support/user/account/updateAvatar.ts new file mode 100644 index 000000000000..d199c3b7374b --- /dev/null +++ b/projects/app/src/pages/api/support/user/account/updateAvatar.ts @@ -0,0 +1,24 @@ +import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import { NextAPI } from '@/service/middleware/entry'; +import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/types'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; + +export type updateAvatarQuery = {}; + +export type updateAvatarBody = { + filename: string; +}; + +export type updateAvatarResponse = CreatePostPresignedUrlResult; + +async function handler( + req: ApiRequestProps, + _: ApiResponseType +): Promise { + const { filename } = req.body; + const { teamId } = await authCert({ req, authToken: true }); + return await getS3AvatarSource().createPostPresignedUrl({ teamId, filename }); +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/system/img/[...id].ts b/projects/app/src/pages/api/system/img/[...id].ts new file mode 100644 index 000000000000..90d0df6be51d --- /dev/null +++ b/projects/app/src/pages/api/system/img/[...id].ts @@ -0,0 +1,32 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import path from 'path'; + +import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; +import { Types } from '@fastgpt/service/common/mongo'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; + +// get the models available to the system +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { id } = req.query as { id: string[] }; + + const joined = id.join('/'); + const parsed = path.parse(joined); + const keys = path.format({ dir: parsed.dir, name: parsed.name, ext: '' }); + + if (Types.ObjectId.isValid(keys)) { + const { binary, mime } = await readMongoImg({ id: joined }); + res.setHeader('Content-Type', mime); + res.send(binary); + return; + } + + res.redirect(301, getS3AvatarSource().createPublicUrl(joined)); + } catch (error) { + jsonRes(res, { + code: 500, + error + }); + } +} diff --git a/projects/app/src/pages/api/system/img/[id].ts b/projects/app/src/pages/api/system/img/[id].ts deleted file mode 100644 index 882d21ab3c87..000000000000 --- a/projects/app/src/pages/api/system/img/[id].ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; - -import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; - -// get the models available to the system -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { id } = req.query as { id: string }; - - const { binary, mime } = await readMongoImg({ id }); - - res.setHeader('Content-Type', mime); - res.send(binary); - } catch (error) { - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/web/common/file/api.ts b/projects/app/src/web/common/file/api.ts index c73dad821095..79094366aecf 100644 --- a/projects/app/src/web/common/file/api.ts +++ b/projects/app/src/web/common/file/api.ts @@ -1,6 +1,6 @@ -import { DELETE, GET, POST } from '@/web/common/api/request'; +import { POST } from '@/web/common/api/request'; import type { UploadImgProps } from '@fastgpt/global/common/file/api.d'; -import type { UploadPresignedURLResponse } from '@fastgpt/service/common/s3/type'; +import type { CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/types'; import { type AxiosProgressEvent } from 'axios'; export const postUploadImg = (e: UploadImgProps) => POST('/common/file/uploadImage', e); @@ -32,3 +32,7 @@ export const postS3UploadFile = ( }, onUploadProgress }); + +export const getUploadAvatarPresignedUrl = (filename: string) => { + return POST('/support/user/account/updateAvatar', { filename }); +}; diff --git a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx b/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx new file mode 100644 index 000000000000..81f92d1c08ed --- /dev/null +++ b/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx @@ -0,0 +1,78 @@ +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; +import { base64ToFile, fileToBase64 } from '@/web/common/file/utils'; +import { compressBase64Img } from '@fastgpt/web/common/file/img'; +import { useToast } from '@fastgpt/web/hooks/useToast'; +import { useCallback, useRef, useState, useTransition } from 'react'; +import { useTranslation } from 'next-i18next'; + +export const useUploadAvatar = (onSuccess: (avatar: string) => void) => { + const { t } = useTranslation(); + const uploadAvatarRef = useRef(null); + const [isUploading, startUpload] = useTransition(); + const { toast } = useToast(); + const [_, setAvatar] = useState(); + + const handleOpenSelectFile = useCallback(() => { + if (!uploadAvatarRef.current) return; + uploadAvatarRef.current.click(); + }, []); + + const onUploadAvatarChange = useCallback( + async (e: React.ChangeEvent) => { + const files = e.target.files; + if (!files || files.length === 0) return; + if (files.length > 1) { + toast({ title: t('account_info:avatar_can_only_select_one'), status: 'warning' }); + return; + } + const file = files[0]; + if (!file) return; + + if (!file.name.match(/\.(jpg|png|jpeg)$/)) { + toast({ title: t('account_info:avatar_can_only_select_jpg_png'), status: 'warning' }); + return; + } + + startUpload(async () => { + const compressed = base64ToFile( + await compressBase64Img({ + base64Img: await fileToBase64(file), + maxW: 300, + maxH: 300 + }), + file.name + ); + const { url, fields } = await getUploadAvatarPresignedUrl(file.name); + const formData = new FormData(); + Object.entries(fields).forEach(([k, v]) => formData.set(k, v)); + formData.set('file', compressed); + await fetch(url, { method: 'POST', body: formData }); // 204 + + const prefix = '/api/system/img/'; + const avatar = `${prefix}${fields.key}`; + setAvatar(avatar); + onSuccess(avatar); + }); + }, + [toast, onSuccess, t] + ); + + const UploadAvatar = () => { + return ( + + ); + }; + + return { + isUploading, + UploadAvatar, + handleOpenSelectFile + }; +}; diff --git a/projects/app/src/web/common/file/utils.ts b/projects/app/src/web/common/file/utils.ts index 160ad29ea3ac..38358902d874 100644 --- a/projects/app/src/web/common/file/utils.ts +++ b/projects/app/src/web/common/file/utils.ts @@ -32,3 +32,15 @@ export const fileToBase64 = (file: File) => { reader.onerror = (error) => reject(error); }); }; + +export const base64ToFile = (base64: string, filename: string) => { + const arr = base64.split(','); + const mime = arr[0].match(/:(.*?);/)?.[1]; + const bstr = atob(arr[1]); + let n = bstr.length; + const u8arr = new Uint8Array(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n); + } + return new File([u8arr], filename, { type: mime }); +}; From cfd01e9c44772328c1a7a1d7332320d791257867 Mon Sep 17 00:00:00 2001 From: xqvvu Date: Fri, 26 Sep 2025 16:15:50 +0800 Subject: [PATCH 2/8] feat: migrate apps' and dataset's avatar to minio --- packages/service/common/s3/buckets/base.ts | 53 +++++++----- packages/service/common/s3/buckets/public.ts | 33 +++++--- packages/service/common/s3/helpers.ts | 8 ++ packages/service/common/s3/interface.ts | 15 ++-- packages/service/common/s3/lifecycle.ts | 23 +++++ packages/service/common/s3/sources/avatar.ts | 32 +++++-- packages/service/common/s3/types.ts | 13 ++- packages/service/common/s3/usage-example.ts | 84 ------------------- .../common/Modal/EditResourceModal.tsx | 32 ++++--- projects/app/src/pages/account/info/index.tsx | 18 ++-- .../updateAvatar.ts => common/file/avatar.ts} | 5 +- projects/app/src/pages/api/core/app/update.ts | 13 ++- .../app/src/pages/api/core/dataset/update.ts | 13 ++- .../pages/api/support/user/account/update.ts | 4 +- projects/app/src/web/common/file/api.ts | 4 +- .../web/common/file/hooks/useUploadAvatar.tsx | 51 ++++++----- 16 files changed, 218 insertions(+), 183 deletions(-) create mode 100644 packages/service/common/s3/lifecycle.ts delete mode 100644 packages/service/common/s3/usage-example.ts rename projects/app/src/pages/api/{support/user/account/updateAvatar.ts => common/file/avatar.ts} (90%) diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts index e2b269aeea03..5a9d8c7d248b 100644 --- a/packages/service/common/s3/buckets/base.ts +++ b/packages/service/common/s3/buckets/base.ts @@ -1,13 +1,19 @@ -import { Client } from 'minio'; +import { Client, type RemoveOptions, type CopyConditions, type LifecycleConfig } from 'minio'; import { defaultS3Options, + type CreatePostPresignedUrlOptions, type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3BucketName, type S3Options } from '../types'; import type { IBucketBasicOperations } from '../interface'; -import { createObjectKey, createPresignedUrlExpires, inferContentType } from '../helpers'; +import { + createObjectKey, + createPresignedUrlExpires, + createTempObjectKey, + inferContentType +} from '../helpers'; export class S3BaseBucket implements IBucketBasicOperations { public client: Client; @@ -16,11 +22,11 @@ export class S3BaseBucket implements IBucketBasicOperations { * * @param _bucket the bucket you want to operate * @param options the options for the s3 client - * @param afterInit the function to be called after instantiating the s3 service + * @param afterInits the function to be called after instantiating the s3 service */ constructor( private readonly _bucket: S3BucketName, - private readonly afterInit?: () => Promise | void, + private readonly afterInits?: (() => Promise | void)[], public options: Partial = defaultS3Options ) { options = { ...defaultS3Options, ...options }; @@ -31,45 +37,54 @@ export class S3BaseBucket implements IBucketBasicOperations { if (!(await this.exist())) { await this.client.makeBucket(this._bucket); } - await this.afterInit?.(); + await Promise.all(this.afterInits?.map((afterInit) => afterInit()) ?? []); }; init(); } - async exist(): Promise { - return await this.client.bucketExists(this._bucket); - } - get name(): string { return this._bucket; } - upload(): Promise { - throw new Error('Method not implemented.'); + async move(src: string, dst: string, options?: CopyConditions): Promise { + const bucket = this.name; + await this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options); + return this.client.removeObject(bucket, src); } - download(): Promise { - throw new Error('Method not implemented.'); + copy(src: string, dst: string, options?: CopyConditions): ReturnType { + return this.client.copyObject(this.name, src, dst, options); } - delete(objectKey: string): Promise { - return this.client.removeObject(this._bucket, objectKey); + exist(): Promise { + return this.client.bucketExists(this.name); + } + + delete(objectKey: string, options?: RemoveOptions): Promise { + return this.client.removeObject(this.name, objectKey, options); } get(): Promise { throw new Error('Method not implemented.'); } + lifecycle(): Promise { + return this.client.getBucketLifecycle(this.name); + } + async createPostPresignedUrl( - params: CreatePostPresignedUrlParams + params: CreatePostPresignedUrlParams, + options: CreatePostPresignedUrlOptions = {} ): Promise { - const maxFileSize = this.options.maxFileSize as number; + const { temporay } = options; const contentType = inferContentType(params.filename); + const maxFileSize = this.options.maxFileSize as number; + const key = temporay ? createTempObjectKey(params) : createObjectKey(params); const policy = this.client.newPostPolicy(); - policy.setBucket(this._bucket); + policy.setKey(key); + policy.setBucket(this.name); policy.setContentType(contentType); - policy.setKey(createObjectKey(params)); policy.setContentLengthRange(1, maxFileSize); policy.setExpires(createPresignedUrlExpires(10)); policy.setUserMetaData({ diff --git a/packages/service/common/s3/buckets/public.ts b/packages/service/common/s3/buckets/public.ts index bd592abf9fe1..4843c910dc94 100644 --- a/packages/service/common/s3/buckets/public.ts +++ b/packages/service/common/s3/buckets/public.ts @@ -2,26 +2,36 @@ import { S3BaseBucket } from './base'; import { createBucketPolicy } from '../helpers'; import { S3Buckets, + type CreatePostPresignedUrlOptions, type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options } from '../types'; import type { IPublicBucketOperations } from '../interface'; +import { lifecycleOfTemporaryAvatars } from '../lifecycle'; export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperations { constructor(options?: Partial) { super( S3Buckets.public, - async () => { - const bucket = this.name; - const policy = createBucketPolicy(bucket); - try { - await this.client.setBucketPolicy(bucket, policy); - } catch (error) { - // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error, - // maybe we can ignore the error, or we have other plan to handle this. + [ + // set bucket policy + async () => { + const bucket = this.name; + const policy = createBucketPolicy(bucket); + try { + await this.client.setBucketPolicy(bucket, policy); + } catch (error) { + // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error, + // maybe we can ignore the error, or we have other plan to handle this. + } + }, + // set bucket lifecycle + async () => { + const bucket = this.name; + await this.client.setBucketLifecycle(bucket, lifecycleOfTemporaryAvatars); } - }, + ], options ); } @@ -36,8 +46,9 @@ export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperati } override createPostPresignedUrl( - params: Omit + params: Omit, + options: CreatePostPresignedUrlOptions = {} ): Promise { - return super.createPostPresignedUrl({ ...params, visibility: 'public' }); + return super.createPostPresignedUrl({ ...params, visibility: 'public' }, options); } } diff --git a/packages/service/common/s3/helpers.ts b/packages/service/common/s3/helpers.ts index c8b5c30f3d98..2b230a3f5342 100644 --- a/packages/service/common/s3/helpers.ts +++ b/packages/service/common/s3/helpers.ts @@ -69,3 +69,11 @@ export function createObjectKey({ source, teamId, filename }: CreateObjectKeyPar const id = crypto.randomBytes(16).toString('hex'); return `${source}/${teamId}/${date}/${id}_${filename}`; } + +/** + * create temporary s3 object key by source, team ID and filename + */ +export function createTempObjectKey(params: CreateObjectKeyParams): string { + const origin = createObjectKey(params); + return `temp/${origin}`; +} diff --git a/packages/service/common/s3/interface.ts b/packages/service/common/s3/interface.ts index 79517eaedfb1..3f99a6fb4c92 100644 --- a/packages/service/common/s3/interface.ts +++ b/packages/service/common/s3/interface.ts @@ -1,14 +1,17 @@ -import type { CreateObjectKeyParams } from './types'; +import { type LifecycleConfig, type Client, type CopyConditions, type RemoveOptions } from 'minio'; +import type { CreateObjectKeyParams, CreatePostPresignedUrlOptions } from './types'; export interface IBucketBasicOperations { get name(): string; - exist(): Promise; - upload(): Promise; - download(): Promise; - delete(objectKey: string): Promise; get(): Promise; + exist(): Promise; + delete(objectKey: string, options?: RemoveOptions): Promise; + move(src: string, dst: string, options?: CopyConditions): Promise; + copy(src: string, dst: string, options?: CopyConditions): ReturnType; + lifecycle(): Promise; createPostPresignedUrl( - params: CreateObjectKeyParams + params: CreateObjectKeyParams, + options?: CreatePostPresignedUrlOptions ): Promise<{ url: string; fields: Record }>; } diff --git a/packages/service/common/s3/lifecycle.ts b/packages/service/common/s3/lifecycle.ts new file mode 100644 index 000000000000..12c28c1ca746 --- /dev/null +++ b/packages/service/common/s3/lifecycle.ts @@ -0,0 +1,23 @@ +import { type LifecycleRule, type LifecycleConfig } from 'minio'; + +export function createLifeCycleConfig(rule: LifecycleRule): LifecycleConfig { + return { + Rule: [rule] + }; +} + +export function assembleLifeCycleConfigs(...configs: LifecycleConfig[]): LifecycleConfig { + if (configs.length === 0) return { Rule: [] }; + return { + Rule: configs.flatMap((config) => config.Rule) + }; +} + +export const lifecycleOfTemporaryAvatars = createLifeCycleConfig({ + ID: 'Temporary Avatars Rule', + Prefix: 'temp/avatar/', + Status: 'Enabled', + Expiration: { + Days: 1 + } +}); diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts index 97f7666a012c..f71d29e472c9 100644 --- a/packages/service/common/s3/sources/avatar.ts +++ b/packages/service/common/s3/sources/avatar.ts @@ -1,6 +1,8 @@ import { S3BaseSource } from './base'; import { S3Sources, + S3APIPrefix, + type CreatePostPresignedUrlOptions, type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options @@ -17,20 +19,36 @@ class S3AvatarSource extends S3BaseSource { } override createPostPresignedUrl( - params: Omit + params: Omit, + options: CreatePostPresignedUrlOptions = {} ): Promise { - return this.bucket.createPostPresignedUrl({ - ...params, - source: S3Sources.avatar - }); + return this.bucket.createPostPresignedUrl( + { + ...params, + source: S3Sources.avatar + }, + options + ); } createPublicUrl(objectKey: string): string { return this.bucket.createPublicUrl(objectKey); } - removeAvatar(objectKey: string): Promise { - return this.bucket.delete(objectKey); + createAvatarObjectKey(avatarWithPrefix: string): string { + return avatarWithPrefix.replace(S3APIPrefix.avatar, ''); + } + + removeAvatar(avatarWithPrefix: string): Promise { + const avatarObjectKey = this.createAvatarObjectKey(avatarWithPrefix); + return this.bucket.delete(avatarObjectKey); + } + + async moveAvatarFromTemp(tempAvatarWithPrefix: string): Promise { + const tempAvatarObjectKey = this.createAvatarObjectKey(tempAvatarWithPrefix); + const avatarObjectKey = tempAvatarObjectKey.replace(`${S3Sources.temp}/`, ''); + await this.bucket.move(tempAvatarObjectKey, avatarObjectKey); + return S3APIPrefix.avatar + avatarObjectKey; } } diff --git a/packages/service/common/s3/types.ts b/packages/service/common/s3/types.ts index b7e66f74bff2..d2be9c357448 100644 --- a/packages/service/common/s3/types.ts +++ b/packages/service/common/s3/types.ts @@ -69,7 +69,8 @@ export const S3SourcesSchema = z.enum([ 'dataset', 'dataset-image', 'invoice', - 'rawtext' + 'rawtext', + 'temp' ]); export const S3Sources = S3SourcesSchema.enum; export type S3SourceType = z.infer; @@ -87,8 +88,18 @@ export const CreatePostPresignedUrlParamsSchema = z.object({ }); export type CreatePostPresignedUrlParams = z.infer; +export const CreatePostPresignedUrlOptionsSchema = z.object({ + temporay: z.boolean().optional() +}); +export type CreatePostPresignedUrlOptions = z.infer; + export const CreatePostPresignedUrlResultSchema = z.object({ url: z.string().min(1), fields: z.record(z.string(), z.string()) }); export type CreatePostPresignedUrlResult = z.infer; + +export const S3APIPrefix = { + avatar: '/api/system/img/' +} as const; +export type S3APIPrefixType = (typeof S3APIPrefix)[keyof typeof S3APIPrefix]; diff --git a/packages/service/common/s3/usage-example.ts b/packages/service/common/s3/usage-example.ts deleted file mode 100644 index 3addcb929090..000000000000 --- a/packages/service/common/s3/usage-example.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * S3 Sources 使用示例 - * 展示如何使用改进后的 S3 架构 - */ - -import { getS3AvatarSource } from './sources/avatar'; -import { getS3ChatSource } from './sources/chat'; - -// 使用示例 -export const s3UsageExample = async () => { - // 获取 avatar source（public bucket） - const avatarSource = getS3AvatarSource(); - - // 获取 chat source（private bucket） - const chatSource = getS3ChatSource(); - - // ✅ 现在可以直接调用 createPublicUrl 方法！ - const publicUrl = avatarSource.createPublicUrl('avatar/team123/2024_01_01/abc123_avatar.png'); - console.log('Avatar public URL:', publicUrl); - - // ✅ 也可以通过 getPublicBucket 获取 bucket 实例 - const publicBucket = avatarSource.getPublicBucket(); - const anotherPublicUrl = publicBucket.createPublicUrl( - 'avatar/team456/2024_01_01/def456_avatar.png' - ); - console.log('Another avatar public URL:', anotherPublicUrl); - - // ✅ 创建预签名 URL - const avatarPresignedUrl = await avatarSource.createPostPresignedUrl({ - filename: 'new-avatar.png', - teamId: '1234567890123456' - }); - console.log('Avatar presigned URL:', avatarPresignedUrl); - - // ✅ Private bucket 示例 - const chatPresignedUrl = await chatSource.createPostPresignedUrl({ - filename: 'chat-history.txt', - teamId: '1234567890123456' - }); - console.log('Chat presigned URL:', chatPresignedUrl); - - // ✅ 获取 private bucket 实例 - const privateBucket = chatSource.getPrivateBucket(); - console.log('Private bucket name:', privateBucket.name); -}; - -// 类型安全示例 -export const typeSafetyExample = () => { - const avatarSource = getS3AvatarSource(); - const chatSource = getS3ChatSource(); - - // ✅ TypeScript 现在知道 avatarSource.bucket 是 S3PublicBucket 类型 - // 所以可以安全地调用 createPublicUrl - const url1 = avatarSource.createPublicUrl('test.png'); - - // ✅ 也可以直接访问 bucket 的方法 - const url2 = avatarSource.bucket.createPublicUrl('test2.png'); - - // ❌ chatSource 没有 createPublicUrl 方法，因为它是 private bucket - // chatSource.createPublicUrl('test.png'); // TypeScript 错误 - - // ✅ 但 chatSource 可以访问基础的 bucket 方法 - console.log('Chat bucket name:', chatSource.bucketName); -}; - -// 单例验证示例 -export const singletonExample = () => { - const avatar1 = getS3AvatarSource(); - const avatar2 = getS3AvatarSource(); - - // 验证单例模式 - console.log('Same instance:', avatar1 === avatar2); // true - console.log('Same bucket:', avatar1.bucket === avatar2.bucket); // true - - const chat1 = getS3ChatSource(); - const chat2 = getS3ChatSource(); - - // 验证单例模式 - console.log('Same chat instance:', chat1 === chat2); // true - console.log('Same chat bucket:', chat1.bucket === chat2.bucket); // true - - // 不同类型的 source 使用不同的 bucket - console.log('Different buckets:', avatar1.bucket !== chat1.bucket); // true -}; diff --git a/projects/app/src/components/common/Modal/EditResourceModal.tsx b/projects/app/src/components/common/Modal/EditResourceModal.tsx index 85c8d06d2c49..5d2114eb2a8a 100644 --- a/projects/app/src/components/common/Modal/EditResourceModal.tsx +++ b/projects/app/src/components/common/Modal/EditResourceModal.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { ModalFooter, ModalBody, Input, Button, Box, Textarea, HStack } from '@chakra-ui/react'; import MyModal from '@fastgpt/web/components/common/MyModal/index'; import { useTranslation } from 'next-i18next'; @@ -8,6 +8,7 @@ import { useForm } from 'react-hook-form'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import Avatar from '@fastgpt/web/components/common/Avatar'; +import { useUploadAvatar } from '@/web/common/file/hooks/useUploadAvatar'; export type EditResourceInfoFormType = { id: string; @@ -41,13 +42,16 @@ const EditResourceModal = ({ } ); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false + const afterUploadAvatar = useCallback( + (avatar: string) => { + console.log('avatar', avatar); + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar({ + temporay: true, + onSuccess: afterUploadAvatar }); return ( @@ -64,7 +68,7 @@ const EditResourceModal = ({ h={'2rem'} cursor={'pointer'} borderRadius={'sm'} - onClick={onOpenSelectFile} + onClick={handleFileSelectorOpen} /> - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> + ); }; diff --git a/projects/app/src/pages/account/info/index.tsx b/projects/app/src/pages/account/info/index.tsx index c08dab4d5bfd..1df7947ad238 100644 --- a/projects/app/src/pages/account/info/index.tsx +++ b/projects/app/src/pages/account/info/index.tsx @@ -157,9 +157,15 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { [reset, t, toast, updateUserInfo] ); - const { UploadAvatar, handleOpenSelectFile } = useUploadAvatar((avatar) => { - if (!userInfo) return; - onclickSave({ ...userInfo, avatar }); + const afterUploadAvatar = useCallback( + (avatar: string) => { + if (!userInfo) return; + onclickSave({ ...userInfo, avatar }); + }, + [onclickSave, userInfo] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar({ + onSuccess: afterUploadAvatar }); const labelStyles: BoxProps = { @@ -239,7 +245,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} - + {isPc ? ( {t('account_info:avatar')}  @@ -252,7 +258,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { border={theme.borders.base} overflow={'hidden'} boxShadow={'0 0 5px rgba(0,0,0,0.1)'} - onClick={handleOpenSelectFile} + onClick={handleFileSelectorOpen} > @@ -263,7 +269,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { flexDirection={'column'} alignItems={'center'} cursor={'pointer'} - onClick={handleOpenSelectFile} + onClick={handleFileSelectorOpen} > , _: ApiResponseType ): Promise { - const { filename } = req.body; + const { filename, temporay } = req.body; const { teamId } = await authCert({ req, authToken: true }); - return await getS3AvatarSource().createPostPresignedUrl({ teamId, filename }); + return await getS3AvatarSource().createPostPresignedUrl({ teamId, filename }, { temporay }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/app/update.ts b/projects/app/src/pages/api/core/app/update.ts index 5709f8ccc7f9..0ab2cd802092 100644 --- a/projects/app/src/pages/api/core/app/update.ts +++ b/projects/app/src/pages/api/core/app/update.ts @@ -27,6 +27,7 @@ import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nAppType } from '@fastgpt/service/support/user/audit/util'; import { i18nT } from '@fastgpt/web/i18n/utils'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; export type AppUpdateQuery = { appId: string; @@ -111,13 +112,19 @@ async function handler(req: ApiRequestProps) { nodes }); - await refreshSourceAvatar(avatar, app.avatar, session); + const permanentAvatar = await (async () => { + if (avatar) { + const s3AvatarSource = getS3AvatarSource(); + await s3AvatarSource.removeAvatar(app.avatar); // 移除旧的头像 + return await s3AvatarSource.moveAvatarFromTemp(avatar); // 永久化临时的预览头像 + } + })(); if (app.type === AppTypeEnum.toolSet && avatar) { await MongoApp.updateMany( { parentId: appId, teamId: app.teamId }, { - avatar + avatar: permanentAvatar }, { session } ); @@ -129,7 +136,7 @@ async function handler(req: ApiRequestProps) { ...parseParentIdInMongo(parentId), ...(name && { name }), ...(type && { type }), - ...(avatar && { avatar }), + ...(avatar && { avatar: permanentAvatar }), ...(intro !== undefined && { intro }), ...(teamTags && { teamTags }), ...(nodes && { diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts index a2b0611ad353..5cd9ad7d51e7 100644 --- a/projects/app/src/pages/api/core/dataset/update.ts +++ b/projects/app/src/pages/api/core/dataset/update.ts @@ -39,6 +39,7 @@ import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model'; import { computedCollectionChunkSettings } from '@fastgpt/global/core/dataset/training/utils'; import { getResourceOwnedClbs } from '@fastgpt/service/support/permission/controller'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; export type DatasetUpdateQuery = {}; export type DatasetUpdateResponse = any; @@ -202,12 +203,20 @@ async function handler( return flattenObjectWithConditions(apiDatasetServer); })(); + const permanentAvatar = await (async () => { + if (avatar) { + const s3AvatarSource = getS3AvatarSource(); + await s3AvatarSource.removeAvatar(dataset.avatar); // 移除旧的头像 + return await s3AvatarSource.moveAvatarFromTemp(avatar); // 永久化临时的预览头像 + } + })(); + await MongoDataset.findByIdAndUpdate( id, { ...parseParentIdInMongo(parentId), ...(name && { name }), - ...(avatar && { avatar }), + ...(avatar && { avatar: permanentAvatar }), ...(agentModel && { agentModel }), ...(vlmModel && { vlmModel }), ...(websiteConfig && { websiteConfig }), @@ -224,8 +233,6 @@ async function handler( dataset, autoSync }); - - await refreshSourceAvatar(avatar, dataset.avatar, session); }; await mongoSessionRun(async (session) => { diff --git a/projects/app/src/pages/api/support/user/account/update.ts b/projects/app/src/pages/api/support/user/account/update.ts index 894772ed65bb..89da6a801f1e 100644 --- a/projects/app/src/pages/api/support/user/account/update.ts +++ b/projects/app/src/pages/api/support/user/account/update.ts @@ -39,9 +39,9 @@ async function handler( // if avatar, update team member avatar if (avatar) { if (tmb?.avatar) { - const objectKey = tmb.avatar.split('/').slice(4).join('/'); - await getS3AvatarSource().removeAvatar(objectKey); + await getS3AvatarSource().removeAvatar(tmb.avatar); } + await MongoTeamMember.updateOne( { _id: tmbId diff --git a/projects/app/src/web/common/file/api.ts b/projects/app/src/web/common/file/api.ts index 79094366aecf..60bed8f0014f 100644 --- a/projects/app/src/web/common/file/api.ts +++ b/projects/app/src/web/common/file/api.ts @@ -33,6 +33,6 @@ export const postS3UploadFile = ( onUploadProgress }); -export const getUploadAvatarPresignedUrl = (filename: string) => { - return POST('/support/user/account/updateAvatar', { filename }); +export const getUploadAvatarPresignedUrl = (params: { filename: string; temporay: boolean }) => { + return POST('/common/file/avatar', params); }; diff --git a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx b/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx index 81f92d1c08ed..fa90cbe32977 100644 --- a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx +++ b/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx @@ -2,17 +2,22 @@ import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; import { base64ToFile, fileToBase64 } from '@/web/common/file/utils'; import { compressBase64Img } from '@fastgpt/web/common/file/img'; import { useToast } from '@fastgpt/web/hooks/useToast'; -import { useCallback, useRef, useState, useTransition } from 'react'; +import { useCallback, useRef, useTransition } from 'react'; import { useTranslation } from 'next-i18next'; -export const useUploadAvatar = (onSuccess: (avatar: string) => void) => { +export const useUploadAvatar = ({ + temporay = false, + onSuccess +}: { + temporay?: boolean; + onSuccess: (avatar: string) => void; +}) => { + const { toast } = useToast(); const { t } = useTranslation(); + const [uploading, startUpload] = useTransition(); const uploadAvatarRef = useRef(null); - const [isUploading, startUpload] = useTransition(); - const { toast } = useToast(); - const [_, setAvatar] = useState(); - const handleOpenSelectFile = useCallback(() => { + const handleFileSelectorOpen = useCallback(() => { if (!uploadAvatarRef.current) return; uploadAvatarRef.current.click(); }, []); @@ -20,16 +25,21 @@ export const useUploadAvatar = (onSuccess: (avatar: string) => void) => { const onUploadAvatarChange = useCallback( async (e: React.ChangeEvent) => { const files = e.target.files; - if (!files || files.length === 0) return; + + if (!files || files.length === 0) { + e.target.value = ''; + return; + } if (files.length > 1) { toast({ title: t('account_info:avatar_can_only_select_one'), status: 'warning' }); + e.target.value = ''; return; } - const file = files[0]; - if (!file) return; + const file = files[0]!; if (!file.name.match(/\.(jpg|png|jpeg)$/)) { toast({ title: t('account_info:avatar_can_only_select_jpg_png'), status: 'warning' }); + e.target.value = ''; return; } @@ -42,22 +52,25 @@ export const useUploadAvatar = (onSuccess: (avatar: string) => void) => { }), file.name ); - const { url, fields } = await getUploadAvatarPresignedUrl(file.name); + const { url, fields } = await getUploadAvatarPresignedUrl({ + filename: file.name, + temporay + }); const formData = new FormData(); Object.entries(fields).forEach(([k, v]) => formData.set(k, v)); formData.set('file', compressed); await fetch(url, { method: 'POST', body: formData }); // 204 - const prefix = '/api/system/img/'; - const avatar = `${prefix}${fields.key}`; - setAvatar(avatar); + const avatar = `/api/system/img/${fields.key}`; onSuccess(avatar); + + e.target.value = ''; }); }, - [toast, onSuccess, t] + [t, temporay, toast, onSuccess] ); - const UploadAvatar = () => { + const Component = useCallback(() => { return ( void) => { onChange={onUploadAvatarChange} /> ); - }; + }, [onUploadAvatarChange]); return { - isUploading, - UploadAvatar, - handleOpenSelectFile + uploading, + Component, + handleFileSelectorOpen }; }; From 7f587868eb8977fd6a70628e952c36eb1dafc121 Mon Sep 17 00:00:00 2001 From: xqvvu Date: Mon, 29 Sep 2025 15:31:06 +0800 Subject: [PATCH 3/8] feat: migrate more avatars to minio --- .../web/common/file/hooks/useUploadAvatar.tsx | 34 ++++++++-------- packages/web/common/file/utils.ts | 21 ++++++++++ .../common/Modal/EditResourceModal.tsx | 15 ++++--- .../account/team/EditInfoModal.tsx | 39 +++++++++---------- projects/app/src/pages/account/info/index.tsx | 12 ++++-- 5 files changed, 73 insertions(+), 48 deletions(-) rename {projects/app/src => packages}/web/common/file/hooks/useUploadAvatar.tsx (75%) diff --git a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx b/packages/web/common/file/hooks/useUploadAvatar.tsx similarity index 75% rename from projects/app/src/web/common/file/hooks/useUploadAvatar.tsx rename to packages/web/common/file/hooks/useUploadAvatar.tsx index fa90cbe32977..10bf1ef04897 100644 --- a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx +++ b/packages/web/common/file/hooks/useUploadAvatar.tsx @@ -1,17 +1,20 @@ -import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; -import { base64ToFile, fileToBase64 } from '@/web/common/file/utils'; -import { compressBase64Img } from '@fastgpt/web/common/file/img'; -import { useToast } from '@fastgpt/web/hooks/useToast'; +import { base64ToFile, fileToBase64 } from '../utils'; +import { compressBase64Img } from '../img'; +import { useToast } from '../../../hooks/useToast'; import { useCallback, useRef, useTransition } from 'react'; import { useTranslation } from 'next-i18next'; +import { type CreatePostPresignedUrlResult } from '../../../../service/common/s3/types'; -export const useUploadAvatar = ({ - temporay = false, - onSuccess -}: { - temporay?: boolean; - onSuccess: (avatar: string) => void; -}) => { +export const useUploadAvatar = ( + api: (params: { filename: string; temporay: boolean }) => Promise, + { + temporay = false, + onSuccess + }: { + temporay?: boolean; + onSuccess?: (avatar: string) => void; + } = {} +) => { const { toast } = useToast(); const { t } = useTranslation(); const [uploading, startUpload] = useTransition(); @@ -52,22 +55,19 @@ export const useUploadAvatar = ({ }), file.name ); - const { url, fields } = await getUploadAvatarPresignedUrl({ - filename: file.name, - temporay - }); + const { url, fields } = await api({ filename: file.name, temporay }); const formData = new FormData(); Object.entries(fields).forEach(([k, v]) => formData.set(k, v)); formData.set('file', compressed); await fetch(url, { method: 'POST', body: formData }); // 204 const avatar = `/api/system/img/${fields.key}`; - onSuccess(avatar); + onSuccess?.(avatar); e.target.value = ''; }); }, - [t, temporay, toast, onSuccess] + [t, temporay, toast, onSuccess, api] ); const Component = useCallback(() => { diff --git a/packages/web/common/file/utils.ts b/packages/web/common/file/utils.ts index 7fd10744530f..f4107301d585 100644 --- a/packages/web/common/file/utils.ts +++ b/packages/web/common/file/utils.ts @@ -99,3 +99,24 @@ async function detectFileEncoding(file: File): Promise { return encoding || 'utf-8'; } + +export const fileToBase64 = (file: File) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => resolve(reader.result as string); + reader.onerror = (error) => reject(error); + }); +}; + +export const base64ToFile = (base64: string, filename: string) => { + const arr = base64.split(','); + const mime = arr[0].match(/:(.*?);/)?.[1]; + const bstr = atob(arr[1]); + let n = bstr.length; + const u8arr = new Uint8Array(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n); + } + return new File([u8arr], filename, { type: mime }); +}; diff --git a/projects/app/src/components/common/Modal/EditResourceModal.tsx b/projects/app/src/components/common/Modal/EditResourceModal.tsx index 5d2114eb2a8a..9827e2bdadc1 100644 --- a/projects/app/src/components/common/Modal/EditResourceModal.tsx +++ b/projects/app/src/components/common/Modal/EditResourceModal.tsx @@ -5,10 +5,10 @@ import { useTranslation } from 'next-i18next'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; import { useForm } from 'react-hook-form'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import Avatar from '@fastgpt/web/components/common/Avatar'; -import { useUploadAvatar } from '@/web/common/file/hooks/useUploadAvatar'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; export type EditResourceInfoFormType = { id: string; @@ -49,10 +49,13 @@ const EditResourceModal = ({ }, [setValue] ); - const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar({ - temporay: true, - onSuccess: afterUploadAvatar - }); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + temporay: true, + onSuccess: afterUploadAvatar + } + ); return ( diff --git a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx index db6f4574b403..0e0a0e95a114 100644 --- a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx +++ b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx @@ -1,7 +1,6 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { useForm } from 'react-hook-form'; import { useTranslation } from 'next-i18next'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { @@ -21,6 +20,8 @@ import { type CreateTeamProps } from '@fastgpt/global/support/user/team/controll import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants'; import Icon from '@fastgpt/web/components/common/Icon'; import dynamic from 'next/dynamic'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; const UpdateContact = dynamic(() => import('@/components/support/user/inform/UpdateContactModal')); export type EditTeamFormDataType = CreateTeamProps & { @@ -50,15 +51,6 @@ function EditModal({ const avatar = watch('avatar'); const notificationAccount = watch('notificationAccount'); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png,.svg', - multiple: false - }); - const { mutate: onclickCreate, isLoading: creating } = useRequest({ mutationFn: async (data: CreateTeamProps) => { return postCreateTeam(data); @@ -88,6 +80,19 @@ function EditModal({ const { isOpen: isOpenContact, onClose: onCloseContact, onOpen: onOpenContact } = useDisclosure(); + const afterUploadAvatar = useCallback( + (avatar: string) => { + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); + return ( {t('account_team:set_name_avatar')} + )} - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> {isOpenContact && ( import('@/pageComponents/account/info/RedeemCouponModal'), { ssr: false @@ -164,9 +165,12 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { }, [onclickSave, userInfo] ); - const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar({ - onSuccess: afterUploadAvatar - }); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); const labelStyles: BoxProps = { flex: '0 0 80px', From 892b7e1be546661b0b6f22c8640659ecf7073560 Mon Sep 17 00:00:00 2001 From: xqvvu Date: Mon, 29 Sep 2025 15:37:01 +0800 Subject: [PATCH 4/8] fix: lock file --- pnpm-lock.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7da780021d45..066581a4aea0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -282,6 +282,9 @@ importers: winston: specifier: ^3.17.0 version: 3.17.0 + zod: + specifier: ^3.24.2 + version: 3.25.51 devDependencies: '@types/cookie': specifier: ^0.5.2 @@ -12231,8 +12234,8 @@ snapshots: express-rate-limit: 7.5.0(express@5.1.0) pkce-challenge: 5.0.0 raw-body: 3.0.0 - zod: 3.24.2 - zod-to-json-schema: 3.24.5(zod@3.24.2) + zod: 3.25.51 + zod-to-json-schema: 3.24.5(zod@3.25.51) transitivePeerDependencies: - supports-color @@ -21900,9 +21903,9 @@ snapshots: - terser - typescript - zod-to-json-schema@3.24.5(zod@3.24.2): + zod-to-json-schema@3.24.5(zod@3.25.51): dependencies: - zod: 3.24.2 + zod: 3.25.51 zod-validation-error@3.4.0(zod@3.25.51): dependencies: From c7d3cc812ddb9bd17cbca06d55ad8c5ccac4d096 Mon Sep 17 00:00:00 2001 From: xqvvu Date: Tue, 30 Sep 2025 15:49:11 +0800 Subject: [PATCH 5/8] feat: migrate copyright settings' logo to minio --- packages/service/common/s3/buckets/manager.ts | 1 - .../web/common/file/hooks/useUploadAvatar.tsx | 43 +++++----- .../ImageUpload/hooks/useImageUpload.tsx | 81 ++++++++++--------- .../chat/ChatSetting/ImageUpload/index.tsx | 2 +- 4 files changed, 67 insertions(+), 60 deletions(-) diff --git a/packages/service/common/s3/buckets/manager.ts b/packages/service/common/s3/buckets/manager.ts index b3f40d0a6c69..a2c546e68e14 100644 --- a/packages/service/common/s3/buckets/manager.ts +++ b/packages/service/common/s3/buckets/manager.ts @@ -1,4 +1,3 @@ -import { type S3BaseBucket } from './base'; import { type S3Options } from '../types'; import { S3PublicBucket } from './public'; import { S3PrivateBucket } from './private'; diff --git a/packages/web/common/file/hooks/useUploadAvatar.tsx b/packages/web/common/file/hooks/useUploadAvatar.tsx index 10bf1ef04897..4b27b026c77f 100644 --- a/packages/web/common/file/hooks/useUploadAvatar.tsx +++ b/packages/web/common/file/hooks/useUploadAvatar.tsx @@ -25,24 +25,11 @@ export const useUploadAvatar = ( uploadAvatarRef.current.click(); }, []); - const onUploadAvatarChange = useCallback( - async (e: React.ChangeEvent) => { - const files = e.target.files; - - if (!files || files.length === 0) { - e.target.value = ''; - return; - } - if (files.length > 1) { - toast({ title: t('account_info:avatar_can_only_select_one'), status: 'warning' }); - e.target.value = ''; - return; - } - const file = files[0]!; - + // manually upload avatar + const handleUploadAvatar = useCallback( + async (file: File) => { if (!file.name.match(/\.(jpg|png|jpeg)$/)) { toast({ title: t('account_info:avatar_can_only_select_jpg_png'), status: 'warning' }); - e.target.value = ''; return; } @@ -63,11 +50,28 @@ export const useUploadAvatar = ( const avatar = `/api/system/img/${fields.key}`; onSuccess?.(avatar); + }); + }, + [t, toast, api, temporay, onSuccess] + ); + + const onUploadAvatarChange = useCallback( + async (e: React.ChangeEvent) => { + const files = e.target.files; + if (!files || files.length === 0) { e.target.value = ''; - }); + return; + } + if (files.length > 1) { + toast({ title: t('account_info:avatar_can_only_select_one'), status: 'warning' }); + e.target.value = ''; + return; + } + const file = files[0]!; + handleUploadAvatar(file); }, - [t, temporay, toast, onSuccess, api] + [t, toast, handleUploadAvatar] ); const Component = useCallback(() => { @@ -86,6 +90,7 @@ export const useUploadAvatar = ( return { uploading, Component, - handleFileSelectorOpen + handleFileSelectorOpen, + handleUploadAvatar }; }; diff --git a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/hooks/useImageUpload.tsx b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/hooks/useImageUpload.tsx index d2470cce6513..2c1586b9dc7c 100644 --- a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/hooks/useImageUpload.tsx +++ b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/hooks/useImageUpload.tsx @@ -1,10 +1,12 @@ -import { useState, useRef } from 'react'; +import { useState, useRef, useCallback } from 'react'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { useTranslation } from 'next-i18next'; import { useMemoizedFn } from 'ahooks'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { formatFileSize } from '@fastgpt/global/common/file/tools'; import { useSystemStore } from '@/web/common/system/useSystemStore'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; export type UploadedFileItem = { url: string; @@ -32,47 +34,48 @@ export const useImageUpload = ({ maxSize, onFileSelect }: UseImageUploadProps) = const finalMaxSize = Math.min(configMaxSize, clientLimitMB); const maxSizeBytes = finalMaxSize * 1024 * 1024; + // const { + // File: SelectFileComponent, + // onOpen: onOpenSelectFile, + // onSelectImage, + // loading + // } = useSelectFile({ + // fileType: 'image/*', + // multiple: false, + // maxCount: 1 + // }); + + const afterUploadAvatar = useCallback((avatar: string) => onFileSelect(avatar), [onFileSelect]); const { - File: SelectFileComponent, - onOpen: onOpenSelectFile, - onSelectImage, - loading - } = useSelectFile({ - fileType: 'image/*', - multiple: false, - maxCount: 1 - }); - - // validate file size - const validateFile = useMemoizedFn((file: File): string | null => { - if (file.size > maxSizeBytes) { - return t('chat:setting.copyright.file_size_exceeds_limit', { - maxSize: formatFileSize(maxSizeBytes) - }); - } - return null; + Component: SelectFileComponent, + uploading: loading, + handleFileSelectorOpen: onOpenSelectFile, + handleUploadAvatar: handleFileSelect + } = useUploadAvatar(getUploadAvatarPresignedUrl, { + temporay: true, + onSuccess: afterUploadAvatar }); // handle file select - immediate upload if enabled - const handleFileSelect = useMemoizedFn(async (files: File[]) => { - const file = files[0]; - - const validationError = validateFile(file); - if (validationError) { - toast({ - status: 'warning', - title: validationError - }); - } - - try { - // 立即上传文件，带TTL - const url = await onSelectImage([file], { maxW: 1000, maxH: 1000 }); - onFileSelect(url); - } catch (error) { - console.error('Failed to upload file:', error); - } - }); + // const handleFileSelect = useMemoizedFn(async (files: File[]) => { + // const file = files[0]; + + // const validationError = validateFile(file); + // if (validationError) { + // toast({ + // status: 'warning', + // title: validationError + // }); + // } + + // try { + // // 立即上传文件，带TTL + // const url = await onSelectImage([file], { maxW: 1000, maxH: 1000 }); + // onFileSelect(url); + // } catch (error) { + // console.error('Failed to upload file:', error); + // } + // }); // 拖拽处理 const handleDragEnter = useMemoizedFn((e: React.DragEvent) => { @@ -106,7 +109,7 @@ export const useImageUpload = ({ maxSize, onFileSelect }: UseImageUploadProps) = if (e.dataTransfer.files && e.dataTransfer.files.length > 0) { const files = Array.from(e.dataTransfer.files); - await handleFileSelect(files); + await handleFileSelect(files[0]); } }); diff --git a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx index bf652689ac71..d89119b22304 100644 --- a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx +++ b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx @@ -80,7 +80,7 @@ const ImageUpload = ({ return ( - + Date: Fri, 10 Oct 2025 15:30:50 +0800 Subject: [PATCH 6/8] feat: integrate minio --- .../global/common/file/minioTtl/type.d.ts | 8 ++ .../common/file/minioTtl/controller.ts | 92 +++++++++++++++++++ .../service/common/file/minioTtl/hooks.ts | 29 ++++++ .../service/common/file/minioTtl/schema.ts | 27 ++++++ .../service/common/file/minioTtl/types.ts | 0 packages/service/common/s3/buckets/base.ts | 15 ++- packages/service/common/s3/sources/avatar.ts | 21 ++++- packages/service/common/s3/types.ts | 3 +- .../common/system/timerLock/constants.ts | 3 +- .../app/src/pages/api/common/file/avatar.ts | 5 +- .../app/src/service/common/system/cron.ts | 2 + 11 files changed, 196 insertions(+), 9 deletions(-) create mode 100644 packages/global/common/file/minioTtl/type.d.ts create mode 100644 packages/service/common/file/minioTtl/controller.ts create mode 100644 packages/service/common/file/minioTtl/hooks.ts create mode 100644 packages/service/common/file/minioTtl/schema.ts create mode 100644 packages/service/common/file/minioTtl/types.ts diff --git a/packages/global/common/file/minioTtl/type.d.ts b/packages/global/common/file/minioTtl/type.d.ts new file mode 100644 index 000000000000..b3b076cd863e --- /dev/null +++ b/packages/global/common/file/minioTtl/type.d.ts @@ -0,0 +1,8 @@ +import type { TeamCollectionName } from '../../../support/user/team/constant'; + +export type MinioTtlSchemaType = { + _id: string; + bucketName: string; + minioKey: string; + expiredTime?: Date; +}; diff --git a/packages/service/common/file/minioTtl/controller.ts b/packages/service/common/file/minioTtl/controller.ts new file mode 100644 index 000000000000..358a2ebb3601 --- /dev/null +++ b/packages/service/common/file/minioTtl/controller.ts @@ -0,0 +1,92 @@ +import { MongoMinioTtl } from './schema'; +import { S3BucketManager } from '../../s3/buckets/manager'; +import { addLog } from '../../system/log'; +import { setCron } from '../../system/cron'; +import { checkTimerLock } from '../../system/timerLock/utils'; +import { TimerIdEnum } from '../../system/timerLock/constants'; + +export async function clearExpiredMinioFiles() { + try { + const now = new Date(); + + const expiredFiles = await MongoMinioTtl.find({ + expiredTime: { $exists: true, $ne: null, $lte: now } + }).lean(); + + if (expiredFiles.length === 0) { + addLog.info('No expired minio files to clean'); + return; + } + + addLog.info(`Found ${expiredFiles.length} expired minio files to clean`); + + const s3Manager = S3BucketManager.getInstance(); + let success = 0; + let fail = 0; + + for (const file of expiredFiles) { + try { + const bucket = (() => { + switch (file.bucketName) { + case process.env.S3_PUBLIC_BUCKET: + return s3Manager.getPublicBucket(); + case process.env.S3_PRIVATE_BUCKET: + return s3Manager.getPrivateBucket(); + default: + throw new Error(`Unknown bucket name: ${file.bucketName}`); + } + })(); + + await bucket.delete(file.minioKey); + + await MongoMinioTtl.deleteOne({ _id: file._id }); + + success++; + addLog.info(`Deleted expired minio file: ${file.minioKey} from bucket: ${file.bucketName}`); + } catch (error) { + fail++; + addLog.error(`Failed to delete minio file: ${file.minioKey}`, error); + } + } + + addLog.info(`Minio TTL cleanup completed. Success: ${success}, Failed: ${fail}`); + } catch (error) { + addLog.error('Error in clearExpiredMinioFiles', error); + } +} + +export function clearExpiredMinioFilesCron() { + // 每小时执行一次 + setCron('0 */1 * * *', async () => { + if ( + await checkTimerLock({ + timerId: TimerIdEnum.clearExpiredMinioFiles, + lockMinuted: 59 + }) + ) { + await clearExpiredMinioFiles(); + } + }); +} + +export async function addMinioTtlFile({ + bucketName, + minioKey, + expiredTime +}: { + bucketName: string; + minioKey: string; + expiredTime?: Date; +}) { + try { + await MongoMinioTtl.create({ + bucketName, + minioKey, + expiredTime + }); + addLog.info(`Added minio TTL file: ${minioKey}, expiredTime: ${expiredTime}`); + } catch (error) { + addLog.error('Failed to add minio TTL file', error); + throw error; + } +} diff --git a/packages/service/common/file/minioTtl/hooks.ts b/packages/service/common/file/minioTtl/hooks.ts new file mode 100644 index 000000000000..43685d847356 --- /dev/null +++ b/packages/service/common/file/minioTtl/hooks.ts @@ -0,0 +1,29 @@ +import { addMinioTtlFile } from './controller'; +import { addLog } from '../../system/log'; + +/** + * @param bucketName - S3 bucket 名称 + * @param objectKey - S3 对象 key + * @param temporay - 是否为临时文件 + * @param ttl - TTL（单位：小时，仅临时文件有效），默认 7 天 + */ +export async function afterCreatePresignedUrl({ + bucketName, + objectKey, + temporay = false, + ttl = 7 * 24 +}: { + bucketName: string; + objectKey: string; + temporay?: boolean; + ttl?: number; +}) { + try { + const expiredTime = temporay ? new Date(Date.now() + ttl * 3.6e6) : undefined; + const info = `TTL: Registered ${temporay ? 'temporary' : 'permanent'} file: ${objectKey}${temporay ? `, expires in ${ttl} hours` : ''}`; + await addMinioTtlFile({ bucketName, expiredTime, minioKey: objectKey }); + addLog.info(info); + } catch (error) { + addLog.error('Failed to register minio TTL', error); + } +} diff --git a/packages/service/common/file/minioTtl/schema.ts b/packages/service/common/file/minioTtl/schema.ts new file mode 100644 index 000000000000..1be867470790 --- /dev/null +++ b/packages/service/common/file/minioTtl/schema.ts @@ -0,0 +1,27 @@ +import { Schema, getMongoModel } from '../../../common/mongo'; +import { type MinioTtlSchemaType } from '@fastgpt/global/common/file/minioTtl/type.d'; + +const collectionName = 'minio_ttl_files'; + +const MinioTtlSchema = new Schema({ + bucketName: { + type: String, + required: true + }, + minioKey: { + type: String, + required: true + }, + expiredTime: { + type: Date + } +}); + +try { + MinioTtlSchema.index({ expiredTime: 1 }); + MinioTtlSchema.index({ bucketName: 1, minioKey: 1 }); +} catch (error) { + console.log(error); +} + +export const MongoMinioTtl = getMongoModel(collectionName, MinioTtlSchema); diff --git a/packages/service/common/file/minioTtl/types.ts b/packages/service/common/file/minioTtl/types.ts new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts index 5a9d8c7d248b..b21d797340ee 100644 --- a/packages/service/common/s3/buckets/base.ts +++ b/packages/service/common/s3/buckets/base.ts @@ -14,6 +14,7 @@ import { createTempObjectKey, inferContentType } from '../helpers'; +import { afterCreatePresignedUrl } from '../../file/minioTtl/hooks'; export class S3BaseBucket implements IBucketBasicOperations { public client: Client; @@ -49,7 +50,7 @@ export class S3BaseBucket implements IBucketBasicOperations { async move(src: string, dst: string, options?: CopyConditions): Promise { const bucket = this.name; await this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options); - return this.client.removeObject(bucket, src); + await this.delete(src); } copy(src: string, dst: string, options?: CopyConditions): ReturnType { @@ -60,8 +61,8 @@ export class S3BaseBucket implements IBucketBasicOperations { return this.client.bucketExists(this.name); } - delete(objectKey: string, options?: RemoveOptions): Promise { - return this.client.removeObject(this.name, objectKey, options); + async delete(objectKey: string, options?: RemoveOptions): Promise { + await this.client.removeObject(this.name, objectKey, options); } get(): Promise { @@ -76,7 +77,7 @@ export class S3BaseBucket implements IBucketBasicOperations { params: CreatePostPresignedUrlParams, options: CreatePostPresignedUrlOptions = {} ): Promise { - const { temporay } = options; + const { temporay, ttl: ttlDays = 7 } = options; const contentType = inferContentType(params.filename); const maxFileSize = this.options.maxFileSize as number; const key = temporay ? createTempObjectKey(params) : createObjectKey(params); @@ -93,6 +94,12 @@ export class S3BaseBucket implements IBucketBasicOperations { }); const { formData, postURL } = await this.client.presignedPostPolicy(policy); + await afterCreatePresignedUrl({ + bucketName: this.name, + objectKey: key, + temporay, + ttl: ttlDays + }); return { url: postURL, diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts index f71d29e472c9..c5ff0fd5231e 100644 --- a/packages/service/common/s3/sources/avatar.ts +++ b/packages/service/common/s3/sources/avatar.ts @@ -8,6 +8,7 @@ import { type S3Options } from '../types'; import type { S3PublicBucket } from '../buckets/public'; +import { MongoMinioTtl } from '../../file/minioTtl/schema'; class S3AvatarSource extends S3BaseSource { constructor(options?: Partial) { @@ -39,15 +40,31 @@ class S3AvatarSource extends S3BaseSource { return avatarWithPrefix.replace(S3APIPrefix.avatar, ''); } - removeAvatar(avatarWithPrefix: string): Promise { + async removeAvatar(avatarWithPrefix: string): Promise { const avatarObjectKey = this.createAvatarObjectKey(avatarWithPrefix); - return this.bucket.delete(avatarObjectKey); + await MongoMinioTtl.deleteOne({ minioKey: avatarObjectKey, bucketName: this.bucketName }); + await this.bucket.delete(avatarObjectKey); } async moveAvatarFromTemp(tempAvatarWithPrefix: string): Promise { const tempAvatarObjectKey = this.createAvatarObjectKey(tempAvatarWithPrefix); const avatarObjectKey = tempAvatarObjectKey.replace(`${S3Sources.temp}/`, ''); + + try { + const file = await MongoMinioTtl.findOne({ + bucketName: this.bucketName, + minioKey: tempAvatarObjectKey + }); + if (file) { + file.set({ expiredTime: undefined, minioKey: avatarObjectKey }); + await file.save(); + } + } catch (error) { + console.error('Failed to convert TTL to permanent:', error); + } + await this.bucket.move(tempAvatarObjectKey, avatarObjectKey); + return S3APIPrefix.avatar + avatarObjectKey; } } diff --git a/packages/service/common/s3/types.ts b/packages/service/common/s3/types.ts index d2be9c357448..fd6be4263924 100644 --- a/packages/service/common/s3/types.ts +++ b/packages/service/common/s3/types.ts @@ -89,7 +89,8 @@ export const CreatePostPresignedUrlParamsSchema = z.object({ export type CreatePostPresignedUrlParams = z.infer; export const CreatePostPresignedUrlOptionsSchema = z.object({ - temporay: z.boolean().optional() + temporay: z.boolean().optional(), + ttl: z.number().positive().optional() // TTL in Hours, default 7 * 24 }); export type CreatePostPresignedUrlOptions = z.infer; diff --git a/packages/service/common/system/timerLock/constants.ts b/packages/service/common/system/timerLock/constants.ts index 76189686c775..2768d0085a19 100644 --- a/packages/service/common/system/timerLock/constants.ts +++ b/packages/service/common/system/timerLock/constants.ts @@ -8,7 +8,8 @@ export enum TimerIdEnum { notification = 'notification', clearExpiredRawTextBuffer = 'clearExpiredRawTextBuffer', - clearExpiredDatasetImage = 'clearExpiredDatasetImage' + clearExpiredDatasetImage = 'clearExpiredDatasetImage', + clearExpiredMinioFiles = 'clearExpiredMinioFiles' } export enum LockNotificationEnum { diff --git a/projects/app/src/pages/api/common/file/avatar.ts b/projects/app/src/pages/api/common/file/avatar.ts index ca5ab4ea242d..d0b0e1b62ed9 100644 --- a/projects/app/src/pages/api/common/file/avatar.ts +++ b/projects/app/src/pages/api/common/file/avatar.ts @@ -19,7 +19,10 @@ async function handler( ): Promise { const { filename, temporay } = req.body; const { teamId } = await authCert({ req, authToken: true }); - return await getS3AvatarSource().createPostPresignedUrl({ teamId, filename }, { temporay }); + return await getS3AvatarSource().createPostPresignedUrl( + { teamId, filename }, + { temporay, ttl: 1 } + ); } export default NextAPI(handler); diff --git a/projects/app/src/service/common/system/cron.ts b/projects/app/src/service/common/system/cron.ts index 70127a39523c..7df8d292922a 100644 --- a/projects/app/src/service/common/system/cron.ts +++ b/projects/app/src/service/common/system/cron.ts @@ -14,6 +14,7 @@ import { getScheduleTriggerApp } from '@/service/core/app/utils'; import { clearExpiredRawTextBufferCron } from '@fastgpt/service/common/buffer/rawText/controller'; import { clearExpiredDatasetImageCron } from '@fastgpt/service/core/dataset/image/controller'; import { cronRefreshModels } from '@fastgpt/service/core/ai/config/utils'; +import { clearExpiredMinioFilesCron } from '@fastgpt/service/common/file/minioTtl/controller'; // Try to run train every minute const setTrainingQueueCron = () => { @@ -90,4 +91,5 @@ export const startCron = () => { clearExpiredRawTextBufferCron(); clearExpiredDatasetImageCron(); cronRefreshModels(); + clearExpiredMinioFilesCron(); }; From fe3ce1da4be10b300d4150c61554c09e45aa836a Mon Sep 17 00:00:00 2001 From: xqvvu Date: Fri, 10 Oct 2025 22:06:21 +0800 Subject: [PATCH 7/8] chore: improve code --- .../global/common/file/minioTtl/type.d.ts | 6 +-- .../service/common/file/minioTtl/hooks.ts | 29 ------------- .../service/common/file/minioTtl/schema.ts | 27 ------------ .../service/common/file/minioTtl/types.ts | 0 .../file/{minioTtl => s3Ttl}/controller.ts | 42 ++++-------------- packages/service/common/file/s3Ttl/schema.ts | 24 +++++++++++ packages/service/common/s3/buckets/base.ts | 43 ++++++++++--------- packages/service/common/s3/buckets/manager.ts | 2 +- packages/service/common/s3/buckets/private.ts | 2 +- packages/service/common/s3/buckets/public.ts | 14 +++++- packages/service/common/s3/helpers.ts | 23 +--------- packages/service/common/s3/interface.ts | 6 +-- packages/service/common/s3/sources/avatar.ts | 8 ++-- packages/service/common/s3/sources/base.ts | 2 +- packages/service/common/s3/sources/chat.ts | 2 +- .../common/s3/sources/dataset-image.ts | 2 +- packages/service/common/s3/sources/dataset.ts | 2 +- packages/service/common/s3/sources/invoice.ts | 2 +- packages/service/common/s3/sources/rawtext.ts | 2 +- .../service/common/s3/{types.ts => type.ts} | 2 +- .../web/common/file/hooks/useUploadAvatar.tsx | 2 +- .../common/Modal/EditResourceModal.tsx | 10 ++--- .../file/{avatar.ts => updateAvatar.ts} | 2 +- projects/app/src/web/common/file/api.ts | 4 +- 24 files changed, 94 insertions(+), 164 deletions(-) delete mode 100644 packages/service/common/file/minioTtl/hooks.ts delete mode 100644 packages/service/common/file/minioTtl/schema.ts delete mode 100644 packages/service/common/file/minioTtl/types.ts rename packages/service/common/file/{minioTtl => s3Ttl}/controller.ts (63%) create mode 100644 packages/service/common/file/s3Ttl/schema.ts rename packages/service/common/s3/{types.ts => type.ts} (98%) rename projects/app/src/pages/api/common/file/{avatar.ts => updateAvatar.ts} (97%) diff --git a/packages/global/common/file/minioTtl/type.d.ts b/packages/global/common/file/minioTtl/type.d.ts index b3b076cd863e..f9442368d0ca 100644 --- a/packages/global/common/file/minioTtl/type.d.ts +++ b/packages/global/common/file/minioTtl/type.d.ts @@ -1,8 +1,6 @@ -import type { TeamCollectionName } from '../../../support/user/team/constant'; - -export type MinioTtlSchemaType = { +export type S3TtlSchemaType = { _id: string; bucketName: string; minioKey: string; - expiredTime?: Date; + expiredTime: Date; }; diff --git a/packages/service/common/file/minioTtl/hooks.ts b/packages/service/common/file/minioTtl/hooks.ts deleted file mode 100644 index 43685d847356..000000000000 --- a/packages/service/common/file/minioTtl/hooks.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { addMinioTtlFile } from './controller'; -import { addLog } from '../../system/log'; - -/** - * @param bucketName - S3 bucket 名称 - * @param objectKey - S3 对象 key - * @param temporay - 是否为临时文件 - * @param ttl - TTL（单位：小时，仅临时文件有效），默认 7 天 - */ -export async function afterCreatePresignedUrl({ - bucketName, - objectKey, - temporay = false, - ttl = 7 * 24 -}: { - bucketName: string; - objectKey: string; - temporay?: boolean; - ttl?: number; -}) { - try { - const expiredTime = temporay ? new Date(Date.now() + ttl * 3.6e6) : undefined; - const info = `TTL: Registered ${temporay ? 'temporary' : 'permanent'} file: ${objectKey}${temporay ? `, expires in ${ttl} hours` : ''}`; - await addMinioTtlFile({ bucketName, expiredTime, minioKey: objectKey }); - addLog.info(info); - } catch (error) { - addLog.error('Failed to register minio TTL', error); - } -} diff --git a/packages/service/common/file/minioTtl/schema.ts b/packages/service/common/file/minioTtl/schema.ts deleted file mode 100644 index 1be867470790..000000000000 --- a/packages/service/common/file/minioTtl/schema.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Schema, getMongoModel } from '../../../common/mongo'; -import { type MinioTtlSchemaType } from '@fastgpt/global/common/file/minioTtl/type.d'; - -const collectionName = 'minio_ttl_files'; - -const MinioTtlSchema = new Schema({ - bucketName: { - type: String, - required: true - }, - minioKey: { - type: String, - required: true - }, - expiredTime: { - type: Date - } -}); - -try { - MinioTtlSchema.index({ expiredTime: 1 }); - MinioTtlSchema.index({ bucketName: 1, minioKey: 1 }); -} catch (error) { - console.log(error); -} - -export const MongoMinioTtl = getMongoModel(collectionName, MinioTtlSchema); diff --git a/packages/service/common/file/minioTtl/types.ts b/packages/service/common/file/minioTtl/types.ts deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/packages/service/common/file/minioTtl/controller.ts b/packages/service/common/file/s3Ttl/controller.ts similarity index 63% rename from packages/service/common/file/minioTtl/controller.ts rename to packages/service/common/file/s3Ttl/controller.ts index 358a2ebb3601..544a4b5c5c46 100644 --- a/packages/service/common/file/minioTtl/controller.ts +++ b/packages/service/common/file/s3Ttl/controller.ts @@ -1,4 +1,4 @@ -import { MongoMinioTtl } from './schema'; +import { MongoS3Ttl } from './schema'; import { S3BucketManager } from '../../s3/buckets/manager'; import { addLog } from '../../system/log'; import { setCron } from '../../system/cron'; @@ -9,7 +9,7 @@ export async function clearExpiredMinioFiles() { try { const now = new Date(); - const expiredFiles = await MongoMinioTtl.find({ + const expiredFiles = await MongoS3Ttl.find({ expiredTime: { $exists: true, $ne: null, $lte: now } }).lean(); @@ -27,19 +27,17 @@ export async function clearExpiredMinioFiles() { for (const file of expiredFiles) { try { const bucket = (() => { - switch (file.bucketName) { - case process.env.S3_PUBLIC_BUCKET: - return s3Manager.getPublicBucket(); - case process.env.S3_PRIVATE_BUCKET: - return s3Manager.getPrivateBucket(); - default: - throw new Error(`Unknown bucket name: ${file.bucketName}`); + if (file.bucketName === process.env.S3_PUBLIC_BUCKET) { + return s3Manager.getPublicBucket(); } + if (file.bucketName === process.env.S3_PRIVATE_BUCKET) { + return s3Manager.getPrivateBucket(); + } + throw new Error(`Unknown bucket name: ${file.bucketName}`); })(); await bucket.delete(file.minioKey); - - await MongoMinioTtl.deleteOne({ _id: file._id }); + await MongoS3Ttl.deleteOne({ _id: file._id }); success++; addLog.info(`Deleted expired minio file: ${file.minioKey} from bucket: ${file.bucketName}`); @@ -68,25 +66,3 @@ export function clearExpiredMinioFilesCron() { } }); } - -export async function addMinioTtlFile({ - bucketName, - minioKey, - expiredTime -}: { - bucketName: string; - minioKey: string; - expiredTime?: Date; -}) { - try { - await MongoMinioTtl.create({ - bucketName, - minioKey, - expiredTime - }); - addLog.info(`Added minio TTL file: ${minioKey}, expiredTime: ${expiredTime}`); - } catch (error) { - addLog.error('Failed to add minio TTL file', error); - throw error; - } -} diff --git a/packages/service/common/file/s3Ttl/schema.ts b/packages/service/common/file/s3Ttl/schema.ts new file mode 100644 index 000000000000..959c719a6f89 --- /dev/null +++ b/packages/service/common/file/s3Ttl/schema.ts @@ -0,0 +1,24 @@ +import { Schema, getMongoModel } from '../../mongo'; +import { type S3TtlSchemaType } from '@fastgpt/global/common/file/minioTtl/type'; + +const collectionName = 's3_ttl_files'; + +const S3TtlSchema = new Schema({ + bucketName: { + type: String, + required: true + }, + minioKey: { + type: String, + required: true + }, + expiredTime: { + type: Date, + required: true + } +}); + +S3TtlSchema.index({ expiredTime: 1 }); +S3TtlSchema.index({ bucketName: 1, minioKey: 1 }); + +export const MongoS3Ttl = getMongoModel(collectionName, S3TtlSchema); diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts index b21d797340ee..d8ece97f0245 100644 --- a/packages/service/common/s3/buckets/base.ts +++ b/packages/service/common/s3/buckets/base.ts @@ -1,22 +1,20 @@ import { Client, type RemoveOptions, type CopyConditions, type LifecycleConfig } from 'minio'; import { defaultS3Options, + type ExtensionType, + Mimes, type CreatePostPresignedUrlOptions, type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3BucketName, type S3Options -} from '../types'; -import type { IBucketBasicOperations } from '../interface'; -import { - createObjectKey, - createPresignedUrlExpires, - createTempObjectKey, - inferContentType -} from '../helpers'; -import { afterCreatePresignedUrl } from '../../file/minioTtl/hooks'; +} from '../type'; +import type { BucketBasicOperationsType } from '../interface'; +import { createObjectKey, createTempObjectKey } from '../helpers'; +import path from 'node:path'; +import { MongoS3Ttl } from 'common/file/s3Ttl/schema'; -export class S3BaseBucket implements IBucketBasicOperations { +export class S3BaseBucket implements BucketBasicOperationsType { public client: Client; /** @@ -69,7 +67,7 @@ export class S3BaseBucket implements IBucketBasicOperations { throw new Error('Method not implemented.'); } - lifecycle(): Promise { + getLifecycle(): Promise { return this.client.getBucketLifecycle(this.name); } @@ -77,29 +75,32 @@ export class S3BaseBucket implements IBucketBasicOperations { params: CreatePostPresignedUrlParams, options: CreatePostPresignedUrlOptions = {} ): Promise { - const { temporay, ttl: ttlDays = 7 } = options; - const contentType = inferContentType(params.filename); + const { temporary, ttl = 7 * 24 } = options; + const ext = path.extname(params.filename).toLowerCase() as ExtensionType; + const contentType = Mimes[ext] ?? 'application/octet-stream'; const maxFileSize = this.options.maxFileSize as number; - const key = temporay ? createTempObjectKey(params) : createObjectKey(params); + const key = temporary ? createTempObjectKey(params) : createObjectKey(params); const policy = this.client.newPostPolicy(); policy.setKey(key); policy.setBucket(this.name); policy.setContentType(contentType); policy.setContentLengthRange(1, maxFileSize); - policy.setExpires(createPresignedUrlExpires(10)); + policy.setExpires(new Date(Date.now() + 10 * 60 * 1000)); policy.setUserMetaData({ filename: encodeURIComponent(params.filename), visibility: params.visibility }); const { formData, postURL } = await this.client.presignedPostPolicy(policy); - await afterCreatePresignedUrl({ - bucketName: this.name, - objectKey: key, - temporay, - ttl: ttlDays - }); + + if (temporary) { + await MongoS3Ttl.create({ + minioKey: key, + bucketName: this.name, + expiredTime: new Date(Date.now() + ttl * 3.6e6) + }); + } return { url: postURL, diff --git a/packages/service/common/s3/buckets/manager.ts b/packages/service/common/s3/buckets/manager.ts index a2c546e68e14..ac957ab07c47 100644 --- a/packages/service/common/s3/buckets/manager.ts +++ b/packages/service/common/s3/buckets/manager.ts @@ -1,4 +1,4 @@ -import { type S3Options } from '../types'; +import { type S3Options } from '../type'; import { S3PublicBucket } from './public'; import { S3PrivateBucket } from './private'; diff --git a/packages/service/common/s3/buckets/private.ts b/packages/service/common/s3/buckets/private.ts index 380ac1503b50..05c80aab88df 100644 --- a/packages/service/common/s3/buckets/private.ts +++ b/packages/service/common/s3/buckets/private.ts @@ -4,7 +4,7 @@ import { type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options -} from '../types'; +} from '../type'; export class S3PrivateBucket extends S3BaseBucket { constructor(options?: Partial) { diff --git a/packages/service/common/s3/buckets/public.ts b/packages/service/common/s3/buckets/public.ts index 4843c910dc94..e0c493dac706 100644 --- a/packages/service/common/s3/buckets/public.ts +++ b/packages/service/common/s3/buckets/public.ts @@ -6,7 +6,7 @@ import { type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options -} from '../types'; +} from '../type'; import type { IPublicBucketOperations } from '../interface'; import { lifecycleOfTemporaryAvatars } from '../lifecycle'; @@ -42,7 +42,17 @@ export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperati const port = this.options.port; const bucket = this.name; - return `${protocol}://${hostname}:${port}/${bucket}/${objectKey}`; + const url = new URL(`${protocol}://${hostname}:${port}/${bucket}/${objectKey}`); + + if (this.options.externalBaseURL) { + const externalBaseURL = new URL(this.options.externalBaseURL); + + url.port = externalBaseURL.port; + url.hostname = externalBaseURL.hostname; + url.protocol = externalBaseURL.protocol; + } + + return url.toString(); } override createPostPresignedUrl( diff --git a/packages/service/common/s3/helpers.ts b/packages/service/common/s3/helpers.ts index 2b230a3f5342..71814d83bc4b 100644 --- a/packages/service/common/s3/helpers.ts +++ b/packages/service/common/s3/helpers.ts @@ -1,28 +1,7 @@ -import path from 'node:path'; import crypto from 'node:crypto'; -import { type ContentType, type CreateObjectKeyParams, type ExtensionType, Mimes } from './types'; +import { type CreateObjectKeyParams } from './type'; import dayjs from 'dayjs'; -/** - * - * @param filename - * @returns the Content-Type relative to the mime type - */ -export const inferContentType = (filename: string): ContentType | 'application/octet-stream' => { - const ext = path.extname(filename).toLowerCase() as ExtensionType; - return Mimes[ext] ?? 'application/octet-stream'; -}; - -/** - * Generate a date that is `minutes` minutes from now - * - * @param minutes - * @returns the date object - */ -export const createPresignedUrlExpires = (minutes: number): Date => { - return new Date(Date.now() + minutes * 60 * 1_000); -}; - /** * use public policy or just a custom policy * diff --git a/packages/service/common/s3/interface.ts b/packages/service/common/s3/interface.ts index 3f99a6fb4c92..b9910b4179c4 100644 --- a/packages/service/common/s3/interface.ts +++ b/packages/service/common/s3/interface.ts @@ -1,14 +1,14 @@ import { type LifecycleConfig, type Client, type CopyConditions, type RemoveOptions } from 'minio'; -import type { CreateObjectKeyParams, CreatePostPresignedUrlOptions } from './types'; +import type { CreateObjectKeyParams, CreatePostPresignedUrlOptions } from './type'; -export interface IBucketBasicOperations { +export interface BucketBasicOperationsType { get name(): string; get(): Promise; exist(): Promise; delete(objectKey: string, options?: RemoveOptions): Promise; move(src: string, dst: string, options?: CopyConditions): Promise; copy(src: string, dst: string, options?: CopyConditions): ReturnType; - lifecycle(): Promise; + getLifecycle(): Promise; createPostPresignedUrl( params: CreateObjectKeyParams, options?: CreatePostPresignedUrlOptions diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts index c5ff0fd5231e..89cd2c524afd 100644 --- a/packages/service/common/s3/sources/avatar.ts +++ b/packages/service/common/s3/sources/avatar.ts @@ -6,9 +6,9 @@ import { type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options -} from '../types'; +} from '../type'; import type { S3PublicBucket } from '../buckets/public'; -import { MongoMinioTtl } from '../../file/minioTtl/schema'; +import { MongoS3Ttl } from '../../file/s3Ttl/schema'; class S3AvatarSource extends S3BaseSource { constructor(options?: Partial) { @@ -42,7 +42,7 @@ class S3AvatarSource extends S3BaseSource { async removeAvatar(avatarWithPrefix: string): Promise { const avatarObjectKey = this.createAvatarObjectKey(avatarWithPrefix); - await MongoMinioTtl.deleteOne({ minioKey: avatarObjectKey, bucketName: this.bucketName }); + await MongoS3Ttl.deleteOne({ minioKey: avatarObjectKey, bucketName: this.bucketName }); await this.bucket.delete(avatarObjectKey); } @@ -51,7 +51,7 @@ class S3AvatarSource extends S3BaseSource { const avatarObjectKey = tempAvatarObjectKey.replace(`${S3Sources.temp}/`, ''); try { - const file = await MongoMinioTtl.findOne({ + const file = await MongoS3Ttl.findOne({ bucketName: this.bucketName, minioKey: tempAvatarObjectKey }); diff --git a/packages/service/common/s3/sources/base.ts b/packages/service/common/s3/sources/base.ts index a34dd20ffece..98e3e8dcfce4 100644 --- a/packages/service/common/s3/sources/base.ts +++ b/packages/service/common/s3/sources/base.ts @@ -6,7 +6,7 @@ import type { CreatePostPresignedUrlResult, S3Options, S3SourceType -} from '../types'; +} from '../type'; type Bucket = S3PublicBucket | S3PrivateBucket; diff --git a/packages/service/common/s3/sources/chat.ts b/packages/service/common/s3/sources/chat.ts index 13094173d23e..e4ef8914f124 100644 --- a/packages/service/common/s3/sources/chat.ts +++ b/packages/service/common/s3/sources/chat.ts @@ -4,7 +4,7 @@ import { type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options -} from '../types'; +} from '../type'; import type { S3PrivateBucket } from '../buckets/private'; class S3ChatSource extends S3BaseSource { diff --git a/packages/service/common/s3/sources/dataset-image.ts b/packages/service/common/s3/sources/dataset-image.ts index 11c6951b589d..5bf71d58b8c1 100644 --- a/packages/service/common/s3/sources/dataset-image.ts +++ b/packages/service/common/s3/sources/dataset-image.ts @@ -4,7 +4,7 @@ import { type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options -} from '../types'; +} from '../type'; import type { S3PrivateBucket } from '../buckets/private'; class S3DatasetImageSource extends S3BaseSource { diff --git a/packages/service/common/s3/sources/dataset.ts b/packages/service/common/s3/sources/dataset.ts index 58aa507823c3..06a9cd3881f2 100644 --- a/packages/service/common/s3/sources/dataset.ts +++ b/packages/service/common/s3/sources/dataset.ts @@ -4,7 +4,7 @@ import { type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options -} from '../types'; +} from '../type'; import type { S3PrivateBucket } from '../buckets/private'; class S3DatasetSource extends S3BaseSource { diff --git a/packages/service/common/s3/sources/invoice.ts b/packages/service/common/s3/sources/invoice.ts index 453020a122a5..57cc7bfff86c 100644 --- a/packages/service/common/s3/sources/invoice.ts +++ b/packages/service/common/s3/sources/invoice.ts @@ -4,7 +4,7 @@ import { type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options -} from '../types'; +} from '../type'; import type { S3PrivateBucket } from '../buckets/private'; class S3InvoiceSource extends S3BaseSource { diff --git a/packages/service/common/s3/sources/rawtext.ts b/packages/service/common/s3/sources/rawtext.ts index d96792003b34..8b505a32475c 100644 --- a/packages/service/common/s3/sources/rawtext.ts +++ b/packages/service/common/s3/sources/rawtext.ts @@ -4,7 +4,7 @@ import { type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options -} from '../types'; +} from '../type'; import type { S3PrivateBucket } from '../buckets/private'; class S3RawtextSource extends S3BaseSource { diff --git a/packages/service/common/s3/types.ts b/packages/service/common/s3/type.ts similarity index 98% rename from packages/service/common/s3/types.ts rename to packages/service/common/s3/type.ts index fd6be4263924..09e44d52c37f 100644 --- a/packages/service/common/s3/types.ts +++ b/packages/service/common/s3/type.ts @@ -89,7 +89,7 @@ export const CreatePostPresignedUrlParamsSchema = z.object({ export type CreatePostPresignedUrlParams = z.infer; export const CreatePostPresignedUrlOptionsSchema = z.object({ - temporay: z.boolean().optional(), + temporary: z.boolean().optional(), ttl: z.number().positive().optional() // TTL in Hours, default 7 * 24 }); export type CreatePostPresignedUrlOptions = z.infer; diff --git a/packages/web/common/file/hooks/useUploadAvatar.tsx b/packages/web/common/file/hooks/useUploadAvatar.tsx index 4b27b026c77f..10c8525e79e1 100644 --- a/packages/web/common/file/hooks/useUploadAvatar.tsx +++ b/packages/web/common/file/hooks/useUploadAvatar.tsx @@ -3,7 +3,7 @@ import { compressBase64Img } from '../img'; import { useToast } from '../../../hooks/useToast'; import { useCallback, useRef, useTransition } from 'react'; import { useTranslation } from 'next-i18next'; -import { type CreatePostPresignedUrlResult } from '../../../../service/common/s3/types'; +import { type CreatePostPresignedUrlResult } from '../../../../service/common/s3/type'; export const useUploadAvatar = ( api: (params: { filename: string; temporay: boolean }) => Promise, diff --git a/projects/app/src/components/common/Modal/EditResourceModal.tsx b/projects/app/src/components/common/Modal/EditResourceModal.tsx index 9827e2bdadc1..e317311f795f 100644 --- a/projects/app/src/components/common/Modal/EditResourceModal.tsx +++ b/projects/app/src/components/common/Modal/EditResourceModal.tsx @@ -49,13 +49,11 @@ const EditResourceModal = ({ }, [setValue] ); - const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( - getUploadAvatarPresignedUrl, - { + const { Component: AvatarUploader, handleFileSelectorOpen: handleAvatarSelectorOpen } = + useUploadAvatar(getUploadAvatarPresignedUrl, { temporay: true, onSuccess: afterUploadAvatar - } - ); + }); return ( @@ -71,7 +69,7 @@ const EditResourceModal = ({ h={'2rem'} cursor={'pointer'} borderRadius={'sm'} - onClick={handleFileSelectorOpen} + onClick={handleAvatarSelectorOpen} /> POST('/common/file/uploadImage', e); @@ -34,5 +34,5 @@ export const postS3UploadFile = ( }); export const getUploadAvatarPresignedUrl = (params: { filename: string; temporay: boolean }) => { - return POST('/common/file/avatar', params); + return POST('/common/file/updateAvatar', params); }; From 9dbb27048a5ec9e8967b51dd5db54c383dab1568 Mon Sep 17 00:00:00 2001 From: xqvvu Date: Fri, 10 Oct 2025 22:25:27 +0800 Subject: [PATCH 8/8] chore: rename variables --- packages/service/common/file/s3Ttl/controller.ts | 9 +++------ packages/service/common/file/s3Ttl/schema.ts | 2 +- packages/service/common/s3/buckets/base.ts | 4 ++-- packages/service/common/s3/sources/avatar.ts | 6 +++--- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/service/common/file/s3Ttl/controller.ts b/packages/service/common/file/s3Ttl/controller.ts index 544a4b5c5c46..c0cea060b2c6 100644 --- a/packages/service/common/file/s3Ttl/controller.ts +++ b/packages/service/common/file/s3Ttl/controller.ts @@ -1,4 +1,4 @@ -import { MongoS3Ttl } from './schema'; +import { MongoS3TTL } from './schema'; import { S3BucketManager } from '../../s3/buckets/manager'; import { addLog } from '../../system/log'; import { setCron } from '../../system/cron'; @@ -9,10 +9,7 @@ export async function clearExpiredMinioFiles() { try { const now = new Date(); - const expiredFiles = await MongoS3Ttl.find({ - expiredTime: { $exists: true, $ne: null, $lte: now } - }).lean(); - + const expiredFiles = await MongoS3TTL.find({ expiredTime: { $lte: now } }).lean(); if (expiredFiles.length === 0) { addLog.info('No expired minio files to clean'); return; @@ -37,7 +34,7 @@ export async function clearExpiredMinioFiles() { })(); await bucket.delete(file.minioKey); - await MongoS3Ttl.deleteOne({ _id: file._id }); + await MongoS3TTL.deleteOne({ _id: file._id }); success++; addLog.info(`Deleted expired minio file: ${file.minioKey} from bucket: ${file.bucketName}`); diff --git a/packages/service/common/file/s3Ttl/schema.ts b/packages/service/common/file/s3Ttl/schema.ts index 959c719a6f89..2919775b2de6 100644 --- a/packages/service/common/file/s3Ttl/schema.ts +++ b/packages/service/common/file/s3Ttl/schema.ts @@ -21,4 +21,4 @@ const S3TtlSchema = new Schema({ S3TtlSchema.index({ expiredTime: 1 }); S3TtlSchema.index({ bucketName: 1, minioKey: 1 }); -export const MongoS3Ttl = getMongoModel(collectionName, S3TtlSchema); +export const MongoS3TTL = getMongoModel(collectionName, S3TtlSchema); diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts index d8ece97f0245..3a48233aa326 100644 --- a/packages/service/common/s3/buckets/base.ts +++ b/packages/service/common/s3/buckets/base.ts @@ -12,7 +12,7 @@ import { import type { BucketBasicOperationsType } from '../interface'; import { createObjectKey, createTempObjectKey } from '../helpers'; import path from 'node:path'; -import { MongoS3Ttl } from 'common/file/s3Ttl/schema'; +import { MongoS3TTL } from 'common/file/s3TTL/schema'; export class S3BaseBucket implements BucketBasicOperationsType { public client: Client; @@ -95,7 +95,7 @@ export class S3BaseBucket implements BucketBasicOperationsType { const { formData, postURL } = await this.client.presignedPostPolicy(policy); if (temporary) { - await MongoS3Ttl.create({ + await MongoS3TTL.create({ minioKey: key, bucketName: this.name, expiredTime: new Date(Date.now() + ttl * 3.6e6) diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts index 89cd2c524afd..1c57baf4da6b 100644 --- a/packages/service/common/s3/sources/avatar.ts +++ b/packages/service/common/s3/sources/avatar.ts @@ -8,7 +8,7 @@ import { type S3Options } from '../type'; import type { S3PublicBucket } from '../buckets/public'; -import { MongoS3Ttl } from '../../file/s3Ttl/schema'; +import { MongoS3TTL } from '../../file/s3TTL/schema'; class S3AvatarSource extends S3BaseSource { constructor(options?: Partial) { @@ -42,7 +42,7 @@ class S3AvatarSource extends S3BaseSource { async removeAvatar(avatarWithPrefix: string): Promise { const avatarObjectKey = this.createAvatarObjectKey(avatarWithPrefix); - await MongoS3Ttl.deleteOne({ minioKey: avatarObjectKey, bucketName: this.bucketName }); + await MongoS3TTL.deleteOne({ minioKey: avatarObjectKey, bucketName: this.bucketName }); await this.bucket.delete(avatarObjectKey); } @@ -51,7 +51,7 @@ class S3AvatarSource extends S3BaseSource { const avatarObjectKey = tempAvatarObjectKey.replace(`${S3Sources.temp}/`, ''); try { - const file = await MongoS3Ttl.findOne({ + const file = await MongoS3TTL.findOne({ bucketName: this.bucketName, minioKey: tempAvatarObjectKey });