From ca2bb0303ee8646f55e64fc7f7fc0419f536dd35 Mon Sep 17 00:00:00 2001 From: xqvvu Date: Thu, 25 Sep 2025 19:39:28 +0800 Subject: [PATCH 1/4] feat: migrate to minio --- packages/service/common/s3/buckets/base.ts | 87 +++++++++ packages/service/common/s3/buckets/manager.ts | 24 +++ packages/service/common/s3/buckets/private.ts | 19 ++ packages/service/common/s3/buckets/public.ts | 43 +++++ packages/service/common/s3/config.ts | 10 -- packages/service/common/s3/const.ts | 20 --- packages/service/common/s3/controller.ts | 168 ------------------ packages/service/common/s3/helpers.ts | 71 ++++++++ packages/service/common/s3/index.ts | 16 -- packages/service/common/s3/interface.ts | 17 ++ packages/service/common/s3/sources/avatar.ts | 39 ++++ packages/service/common/s3/sources/base.ts | 58 ++++++ packages/service/common/s3/sources/chat.ts | 28 +++ .../common/s3/sources/dataset-image.ts | 28 +++ packages/service/common/s3/sources/dataset.ts | 28 +++ packages/service/common/s3/sources/index.ts | 24 +++ packages/service/common/s3/sources/invoice.ts | 28 +++ packages/service/common/s3/sources/rawtext.ts | 28 +++ packages/service/common/s3/type.ts | 49 ----- packages/service/common/s3/types.ts | 94 ++++++++++ packages/service/common/s3/usage-example.ts | 84 +++++++++ packages/service/package.json | 3 +- packages/web/i18n/en/account_info.json | 2 + packages/web/i18n/zh-CN/account_info.json | 2 + packages/web/i18n/zh-Hant/account_info.json | 2 + projects/app/.env.template | 5 +- projects/app/src/pages/account/info/index.tsx | 35 ++-- .../pages/api/support/user/account/update.ts | 6 +- .../api/support/user/account/updateAvatar.ts | 24 +++ .../app/src/pages/api/system/img/[...id].ts | 32 ++++ projects/app/src/pages/api/system/img/[id].ts | 21 --- projects/app/src/web/common/file/api.ts | 8 +- .../web/common/file/hooks/useUploadAvatar.tsx | 78 ++++++++ projects/app/src/web/common/file/utils.ts | 12 ++ 34 files changed, 879 insertions(+), 314 deletions(-) create mode 100644 packages/service/common/s3/buckets/base.ts create mode 100644 packages/service/common/s3/buckets/manager.ts create mode 100644 packages/service/common/s3/buckets/private.ts create mode 100644 packages/service/common/s3/buckets/public.ts delete mode 100644 packages/service/common/s3/config.ts delete mode 100644 packages/service/common/s3/const.ts delete mode 100644 packages/service/common/s3/controller.ts create mode 100644 packages/service/common/s3/helpers.ts delete mode 100644 packages/service/common/s3/index.ts create mode 100644 packages/service/common/s3/interface.ts create mode 100644 packages/service/common/s3/sources/avatar.ts create mode 100644 packages/service/common/s3/sources/base.ts create mode 100644 packages/service/common/s3/sources/chat.ts create mode 100644 packages/service/common/s3/sources/dataset-image.ts create mode 100644 packages/service/common/s3/sources/dataset.ts create mode 100644 packages/service/common/s3/sources/index.ts create mode 100644 packages/service/common/s3/sources/invoice.ts create mode 100644 packages/service/common/s3/sources/rawtext.ts delete mode 100644 packages/service/common/s3/type.ts create mode 100644 packages/service/common/s3/types.ts create mode 100644 packages/service/common/s3/usage-example.ts create mode 100644 projects/app/src/pages/api/support/user/account/updateAvatar.ts create mode 100644 projects/app/src/pages/api/system/img/[...id].ts delete mode 100644 projects/app/src/pages/api/system/img/[id].ts create mode 100644 projects/app/src/web/common/file/hooks/useUploadAvatar.tsx diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts new file mode 100644 index 000000000000..e2b269aeea03 --- /dev/null +++ b/packages/service/common/s3/buckets/base.ts @@ -0,0 +1,87 @@ +import { Client } from 'minio'; +import { + defaultS3Options, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3BucketName, + type S3Options +} from '../types'; +import type { IBucketBasicOperations } from '../interface'; +import { createObjectKey, createPresignedUrlExpires, inferContentType } from '../helpers'; + +export class S3BaseBucket implements IBucketBasicOperations { + public client: Client; + + /** + * + * @param _bucket the bucket you want to operate + * @param options the options for the s3 client + * @param afterInit the function to be called after instantiating the s3 service + */ + constructor( + private readonly _bucket: S3BucketName, + private readonly afterInit?: () => Promise | void, + public options: Partial = defaultS3Options + ) { + options = { ...defaultS3Options, ...options }; + this.options = options as S3Options; + this.client = new Client(options as S3Options); + + const init = async () => { + if (!(await this.exist())) { + await this.client.makeBucket(this._bucket); + } + await this.afterInit?.(); + }; + init(); + } + + async exist(): Promise { + return await this.client.bucketExists(this._bucket); + } + + get name(): string { + return this._bucket; + } + + upload(): Promise { + throw new Error('Method not implemented.'); + } + + download(): Promise { + throw new Error('Method not implemented.'); + } + + delete(objectKey: string): Promise { + return this.client.removeObject(this._bucket, objectKey); + } + + get(): Promise { + throw new Error('Method not implemented.'); + } + + async createPostPresignedUrl( + params: CreatePostPresignedUrlParams + ): Promise { + const maxFileSize = this.options.maxFileSize as number; + const contentType = inferContentType(params.filename); + + const policy = this.client.newPostPolicy(); + policy.setBucket(this._bucket); + policy.setContentType(contentType); + policy.setKey(createObjectKey(params)); + policy.setContentLengthRange(1, maxFileSize); + policy.setExpires(createPresignedUrlExpires(10)); + policy.setUserMetaData({ + filename: encodeURIComponent(params.filename), + visibility: params.visibility + }); + + const { formData, postURL } = await this.client.presignedPostPolicy(policy); + + return { + url: postURL, + fields: formData + }; + } +} diff --git a/packages/service/common/s3/buckets/manager.ts b/packages/service/common/s3/buckets/manager.ts new file mode 100644 index 000000000000..b3f40d0a6c69 --- /dev/null +++ b/packages/service/common/s3/buckets/manager.ts @@ -0,0 +1,24 @@ +import { type S3BaseBucket } from './base'; +import { type S3Options } from '../types'; +import { S3PublicBucket } from './public'; +import { S3PrivateBucket } from './private'; + +export class S3BucketManager { + private static instance: S3BucketManager; + private publicBucket: S3PublicBucket | null = null; + private privateBucket: S3PrivateBucket | null = null; + + private constructor() {} + + static getInstance(): S3BucketManager { + return (this.instance ??= new S3BucketManager()); + } + + getPublicBucket(options?: Partial): S3PublicBucket { + return (this.publicBucket ??= new S3PublicBucket(options)); + } + + getPrivateBucket(options?: Partial): S3PrivateBucket { + return (this.privateBucket ??= new S3PrivateBucket(options)); + } +} diff --git a/packages/service/common/s3/buckets/private.ts b/packages/service/common/s3/buckets/private.ts new file mode 100644 index 000000000000..380ac1503b50 --- /dev/null +++ b/packages/service/common/s3/buckets/private.ts @@ -0,0 +1,19 @@ +import { S3BaseBucket } from './base'; +import { + S3Buckets, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; + +export class S3PrivateBucket extends S3BaseBucket { + constructor(options?: Partial) { + super(S3Buckets.private, undefined, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return super.createPostPresignedUrl({ ...params, visibility: 'private' }); + } +} diff --git a/packages/service/common/s3/buckets/public.ts b/packages/service/common/s3/buckets/public.ts new file mode 100644 index 000000000000..bd592abf9fe1 --- /dev/null +++ b/packages/service/common/s3/buckets/public.ts @@ -0,0 +1,43 @@ +import { S3BaseBucket } from './base'; +import { createBucketPolicy } from '../helpers'; +import { + S3Buckets, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { IPublicBucketOperations } from '../interface'; + +export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperations { + constructor(options?: Partial) { + super( + S3Buckets.public, + async () => { + const bucket = this.name; + const policy = createBucketPolicy(bucket); + try { + await this.client.setBucketPolicy(bucket, policy); + } catch (error) { + // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error, + // maybe we can ignore the error, or we have other plan to handle this. + } + }, + options + ); + } + + createPublicUrl(objectKey: string): string { + const protocol = this.options.useSSL ? 'https' : 'http'; + const hostname = this.options.endPoint; + const port = this.options.port; + const bucket = this.name; + + return `${protocol}://${hostname}:${port}/${bucket}/${objectKey}`; + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return super.createPostPresignedUrl({ ...params, visibility: 'public' }); + } +} diff --git a/packages/service/common/s3/config.ts b/packages/service/common/s3/config.ts deleted file mode 100644 index 501d61be4b1a..000000000000 --- a/packages/service/common/s3/config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { S3ServiceConfig } from './type'; - -export const defualtS3Config: Omit = { - endPoint: process.env.S3_ENDPOINT || 'localhost', - port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, - useSSL: process.env.S3_USE_SSL === 'true', - accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', - secretKey: process.env.S3_SECRET_KEY || 'minioadmin', - externalBaseURL: process.env.S3_EXTERNAL_BASE_URL -}; diff --git a/packages/service/common/s3/const.ts b/packages/service/common/s3/const.ts deleted file mode 100644 index c91043dc1f96..000000000000 --- a/packages/service/common/s3/const.ts +++ /dev/null @@ -1,20 +0,0 @@ -export const mimeMap: Record = { - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.png': 'image/png', - '.gif': 'image/gif', - '.webp': 'image/webp', - '.svg': 'image/svg+xml', - '.pdf': 'application/pdf', - '.txt': 'text/plain', - '.json': 'application/json', - '.csv': 'text/csv', - '.zip': 'application/zip', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - '.doc': 'application/msword', - '.xls': 'application/vnd.ms-excel', - '.ppt': 'application/vnd.ms-powerpoint', - '.js': 'application/javascript' -}; diff --git a/packages/service/common/s3/controller.ts b/packages/service/common/s3/controller.ts deleted file mode 100644 index faaae7cdc91e..000000000000 --- a/packages/service/common/s3/controller.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { Client } from 'minio'; -import { - type FileMetadataType, - type PresignedUrlInput as UploadPresignedURLProps, - type UploadPresignedURLResponse, - type S3ServiceConfig -} from './type'; -import { defualtS3Config } from './config'; -import { randomBytes } from 'crypto'; -import { HttpProxyAgent } from 'http-proxy-agent'; -import { HttpsProxyAgent } from 'https-proxy-agent'; -import { extname } from 'path'; -import { addLog } from '../../common/system/log'; -import { getErrText } from '@fastgpt/global/common/error/utils'; -import { mimeMap } from './const'; - -export class S3Service { - private client: Client; - private config: S3ServiceConfig; - private initialized: boolean = false; - initFunction?: () => Promise; - - constructor(config?: Partial) { - this.config = { ...defualtS3Config, ...config } as S3ServiceConfig; - - this.client = new Client({ - endPoint: this.config.endPoint, - port: this.config.port, - useSSL: this.config.useSSL, - accessKey: this.config.accessKey, - secretKey: this.config.secretKey, - transportAgent: process.env.HTTP_PROXY - ? new HttpProxyAgent(process.env.HTTP_PROXY) - : process.env.HTTPS_PROXY - ? new HttpsProxyAgent(process.env.HTTPS_PROXY) - : undefined - }); - - this.initFunction = config?.initFunction; - } - - public async init() { - if (!this.initialized) { - if (!(await this.client.bucketExists(this.config.bucket))) { - addLog.debug(`Creating bucket: ${this.config.bucket}`); - await this.client.makeBucket(this.config.bucket); - } - - await this.initFunction?.(); - this.initialized = true; - } - } - - private generateFileId(): string { - return randomBytes(16).toString('hex'); - } - - private generateAccessUrl(filename: string): string { - const protocol = this.config.useSSL ? 'https' : 'http'; - const port = - this.config.port && this.config.port !== (this.config.useSSL ? 443 : 80) - ? `:${this.config.port}` - : ''; - - const externalBaseURL = this.config.externalBaseURL; - return externalBaseURL - ? `${externalBaseURL}/${this.config.bucket}/${encodeURIComponent(filename)}` - : `${protocol}://${this.config.endPoint}${port}/${this.config.bucket}/${encodeURIComponent(filename)}`; - } - - uploadFile = async (fileBuffer: Buffer, originalFilename: string): Promise => { - await this.init(); - const inferContentType = (filename: string) => { - const ext = extname(filename).toLowerCase(); - return mimeMap[ext] || 'application/octet-stream'; - }; - - if (this.config.maxFileSize && fileBuffer.length > this.config.maxFileSize) { - return Promise.reject( - `File size ${fileBuffer.length} exceeds limit ${this.config.maxFileSize}` - ); - } - - const fileId = this.generateFileId(); - const objectName = `${fileId}-${originalFilename}`; - const uploadTime = new Date(); - - const contentType = inferContentType(originalFilename); - await this.client.putObject(this.config.bucket, objectName, fileBuffer, fileBuffer.length, { - 'Content-Type': contentType, - 'Content-Disposition': `attachment; filename="${encodeURIComponent(originalFilename)}"`, - 'x-amz-meta-original-filename': encodeURIComponent(originalFilename), - 'x-amz-meta-upload-time': uploadTime.toISOString() - }); - - const metadata: FileMetadataType = { - fileId, - originalFilename, - contentType, - size: fileBuffer.length, - uploadTime, - accessUrl: this.generateAccessUrl(objectName) - }; - - return metadata; - }; - - generateUploadPresignedURL = async ({ - filepath, - contentType, - metadata, - filename - }: UploadPresignedURLProps): Promise => { - await this.init(); - const objectName = `${filepath}/${filename}`; - - try { - const policy = this.client.newPostPolicy(); - - policy.setBucket(this.config.bucket); - policy.setKey(objectName); - if (contentType) { - policy.setContentType(contentType); - } - if (this.config.maxFileSize) { - policy.setContentLengthRange(1, this.config.maxFileSize); - } - policy.setExpires(new Date(Date.now() + 10 * 60 * 1000)); // 10 mins - - policy.setUserMetaData({ - 'original-filename': encodeURIComponent(filename), - 'upload-time': new Date().toISOString(), - ...metadata - }); - - const { postURL, formData } = await this.client.presignedPostPolicy(policy); - - const response: UploadPresignedURLResponse = { - objectName, - uploadUrl: postURL, - formData - }; - - return response; - } catch (error) { - addLog.error('Failed to generate Upload Presigned URL', error); - return Promise.reject(`Failed to generate Upload Presigned URL: ${getErrText(error)}`); - } - }; - - generateDownloadUrl = (objectName: string): string => { - const pathParts = objectName.split('/'); - const encodedParts = pathParts.map((part) => encodeURIComponent(part)); - const encodedObjectName = encodedParts.join('/'); - return `${this.config.bucket}/${encodedObjectName}`; - }; - - getFile = async (objectName: string): Promise => { - const stat = await this.client.statObject(this.config.bucket, objectName); - - if (stat.size > 0) { - const accessUrl = this.generateDownloadUrl(objectName); - return accessUrl; - } - - return Promise.reject(`File ${objectName} not found`); - }; -} diff --git a/packages/service/common/s3/helpers.ts b/packages/service/common/s3/helpers.ts new file mode 100644 index 000000000000..c8b5c30f3d98 --- /dev/null +++ b/packages/service/common/s3/helpers.ts @@ -0,0 +1,71 @@ +import path from 'node:path'; +import crypto from 'node:crypto'; +import { type ContentType, type CreateObjectKeyParams, type ExtensionType, Mimes } from './types'; +import dayjs from 'dayjs'; + +/** + * + * @param filename + * @returns the Content-Type relative to the mime type + */ +export const inferContentType = (filename: string): ContentType | 'application/octet-stream' => { + const ext = path.extname(filename).toLowerCase() as ExtensionType; + return Mimes[ext] ?? 'application/octet-stream'; +}; + +/** + * Generate a date that is `minutes` minutes from now + * + * @param minutes + * @returns the date object + */ +export const createPresignedUrlExpires = (minutes: number): Date => { + return new Date(Date.now() + minutes * 60 * 1_000); +}; + +/** + * use public policy or just a custom policy + * + * @default policy public policy + * @param bucket bucket name + * @returns the policy string + */ +export const createBucketPolicy = ( + bucket: string, + policy?: 'public' | Record +): string => { + if (typeof policy === 'string' && policy !== 'public') { + throw new Error("'policy' only can be assigned to 'public' if typeof 'policy' is string"); + } + + switch (typeof policy) { + case 'string': + case 'undefined': + default: + return JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: '*', + Action: 's3:GetObject', + Resource: `arn:aws:s3:::${bucket}/*` + } + ] + }); + case 'object': + if (policy === null) { + throw new Error("Bucket policy can't be null"); + } + return JSON.stringify(policy); + } +}; + +/** + * create s3 object key by source, team ID and filename + */ +export function createObjectKey({ source, teamId, filename }: CreateObjectKeyParams): string { + const date = dayjs().format('YYYY_MM_DD'); + const id = crypto.randomBytes(16).toString('hex'); + return `${source}/${teamId}/${date}/${id}_${filename}`; +} diff --git a/packages/service/common/s3/index.ts b/packages/service/common/s3/index.ts deleted file mode 100644 index 761bd564097e..000000000000 --- a/packages/service/common/s3/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { S3Service } from './controller'; - -export const PluginS3Service = (() => { - if (!global.pluginS3Service) { - global.pluginS3Service = new S3Service({ - bucket: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', - maxFileSize: 50 * 1024 * 1024 // 50MB - }); - } - - return global.pluginS3Service; -})(); - -declare global { - var pluginS3Service: S3Service; -} diff --git a/packages/service/common/s3/interface.ts b/packages/service/common/s3/interface.ts new file mode 100644 index 000000000000..79517eaedfb1 --- /dev/null +++ b/packages/service/common/s3/interface.ts @@ -0,0 +1,17 @@ +import type { CreateObjectKeyParams } from './types'; + +export interface IBucketBasicOperations { + get name(): string; + exist(): Promise; + upload(): Promise; + download(): Promise; + delete(objectKey: string): Promise; + get(): Promise; + createPostPresignedUrl( + params: CreateObjectKeyParams + ): Promise<{ url: string; fields: Record }>; +} + +export interface IPublicBucketOperations { + createPublicUrl(objectKey: string): string; +} diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts new file mode 100644 index 000000000000..97f7666a012c --- /dev/null +++ b/packages/service/common/s3/sources/avatar.ts @@ -0,0 +1,39 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PublicBucket } from '../buckets/public'; + +class S3AvatarSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.avatar, true, options); + } + + static getInstance(options?: Partial): S3AvatarSource { + return S3BaseSource._getInstance(S3AvatarSource, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ + ...params, + source: S3Sources.avatar + }); + } + + createPublicUrl(objectKey: string): string { + return this.bucket.createPublicUrl(objectKey); + } + + removeAvatar(objectKey: string): Promise { + return this.bucket.delete(objectKey); + } +} + +export function getS3AvatarSource() { + return S3AvatarSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/base.ts b/packages/service/common/s3/sources/base.ts new file mode 100644 index 000000000000..a34dd20ffece --- /dev/null +++ b/packages/service/common/s3/sources/base.ts @@ -0,0 +1,58 @@ +import { S3BucketManager } from '../buckets/manager'; +import type { S3PrivateBucket } from '../buckets/private'; +import type { S3PublicBucket } from '../buckets/public'; +import type { + CreatePostPresignedUrlParams, + CreatePostPresignedUrlResult, + S3Options, + S3SourceType +} from '../types'; + +type Bucket = S3PublicBucket | S3PrivateBucket; + +export abstract class S3BaseSource { + protected bucket: T; + protected static instances: Map = new Map(); + + constructor( + protected readonly source: S3SourceType, + protected readonly pub: boolean = false, + options?: Partial + ) { + const manager = S3BucketManager.getInstance(); + switch (pub) { + case true: + this.bucket = manager.getPublicBucket(options) as T; + break; + case false: + this.bucket = manager.getPrivateBucket(options) as T; + } + } + + abstract createPostPresignedUrl( + params: Omit + ): Promise; + + protected static _getInstance( + constructor: new (options?: Partial) => T, + options?: Partial + ): T { + const className = constructor.name; + if (!S3BaseSource.instances.has(className)) { + S3BaseSource.instances.set(className, new constructor(options)); + } + return S3BaseSource.instances.get(className) as T; + } + + protected getBucket(): T { + return this.bucket; + } + + get bucketName(): string { + return this.bucket.name; + } + + isBucketExist(): Promise { + return this.bucket.exist(); + } +} diff --git a/packages/service/common/s3/sources/chat.ts b/packages/service/common/s3/sources/chat.ts new file mode 100644 index 000000000000..13094173d23e --- /dev/null +++ b/packages/service/common/s3/sources/chat.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3ChatSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.chat, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.chat }); + } + + static getInstance(options?: Partial): S3ChatSource { + return S3BaseSource._getInstance(S3ChatSource, options); + } +} + +export function getS3ChatSource() { + return S3ChatSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/dataset-image.ts b/packages/service/common/s3/sources/dataset-image.ts new file mode 100644 index 000000000000..11c6951b589d --- /dev/null +++ b/packages/service/common/s3/sources/dataset-image.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3DatasetImageSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources['dataset-image'], false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources['dataset-image'] }); + } + + static getInstance(options?: Partial): S3DatasetImageSource { + return S3BaseSource._getInstance(S3DatasetImageSource, options); + } +} + +export function getS3DatasetImageSource() { + return S3DatasetImageSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/dataset.ts b/packages/service/common/s3/sources/dataset.ts new file mode 100644 index 000000000000..58aa507823c3 --- /dev/null +++ b/packages/service/common/s3/sources/dataset.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3DatasetSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.dataset, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.dataset }); + } + + static getInstance(options?: Partial): S3DatasetSource { + return S3BaseSource._getInstance(S3DatasetSource, options); + } +} + +export function getS3DatasetSource() { + return S3DatasetSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/index.ts b/packages/service/common/s3/sources/index.ts new file mode 100644 index 000000000000..a19622639276 --- /dev/null +++ b/packages/service/common/s3/sources/index.ts @@ -0,0 +1,24 @@ +import { getS3AvatarSource } from './avatar'; +import { getS3ChatSource } from './chat'; +import { getS3DatasetSource } from './dataset'; +import { getS3DatasetImageSource } from './dataset-image'; +import { getS3InvoiceSource } from './invoice'; +import { getS3RawtextSource } from './rawtext'; + +export function registerSources() { + getS3AvatarSource(); + getS3ChatSource(); + getS3DatasetImageSource(); + getS3DatasetSource(); + getS3InvoiceSource(); + getS3RawtextSource(); +} + +export { + getS3AvatarSource, + getS3ChatSource, + getS3DatasetImageSource, + getS3DatasetSource, + getS3InvoiceSource, + getS3RawtextSource +}; diff --git a/packages/service/common/s3/sources/invoice.ts b/packages/service/common/s3/sources/invoice.ts new file mode 100644 index 000000000000..453020a122a5 --- /dev/null +++ b/packages/service/common/s3/sources/invoice.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3InvoiceSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.invoice, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.invoice }); + } + + static getInstance(options?: Partial): S3InvoiceSource { + return S3BaseSource._getInstance(S3InvoiceSource, options); + } +} + +export function getS3InvoiceSource() { + return S3InvoiceSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/rawtext.ts b/packages/service/common/s3/sources/rawtext.ts new file mode 100644 index 000000000000..d96792003b34 --- /dev/null +++ b/packages/service/common/s3/sources/rawtext.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3RawtextSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.rawtext, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.rawtext }); + } + + static getInstance(options?: Partial): S3RawtextSource { + return S3BaseSource._getInstance(S3RawtextSource, options); + } +} + +export function getS3RawtextSource() { + return S3RawtextSource.getInstance(); +} diff --git a/packages/service/common/s3/type.ts b/packages/service/common/s3/type.ts deleted file mode 100644 index a480530493e8..000000000000 --- a/packages/service/common/s3/type.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { ClientOptions } from 'minio'; - -export type S3ServiceConfig = { - bucket: string; - externalBaseURL?: string; - /** - * Unit: Byte - */ - maxFileSize?: number; - /** - * for executing some init function for the s3 service - */ - initFunction?: () => Promise; -} & ClientOptions; - -export type FileMetadataType = { - fileId: string; - originalFilename: string; - contentType: string; - size: number; - uploadTime: Date; - accessUrl: string; -}; - -export type PresignedUrlInput = { - filepath: string; - filename: string; - contentType?: string; - metadata?: Record; -}; - -export type UploadPresignedURLResponse = { - objectName: string; - uploadUrl: string; - formData: Record; -}; - -export type FileUploadInput = { - buffer: Buffer; - filename: string; -}; - -export enum PluginTypeEnum { - tool = 'tool' -} - -export const PluginFilePath = { - [PluginTypeEnum.tool]: 'plugin/tools' -}; diff --git a/packages/service/common/s3/types.ts b/packages/service/common/s3/types.ts new file mode 100644 index 000000000000..b7e66f74bff2 --- /dev/null +++ b/packages/service/common/s3/types.ts @@ -0,0 +1,94 @@ +import type { ClientOptions } from 'minio'; +import { HttpProxyAgent } from 'http-proxy-agent'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import { z } from 'zod'; + +export const S3MetadataSchema = z.object({ + filename: z.string(), + uploadedAt: z.date(), + accessUrl: z.string(), + contentType: z.string(), + id: z.string().length(32), + size: z.number().positive() +}); +export type S3Metadata = z.infer; + +export const Mimes = { + '.gif': 'image/gif', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + + '.csv': 'text/csv', + '.txt': 'text/plain', + + '.pdf': 'application/pdf', + '.zip': 'application/zip', + '.json': 'application/json', + '.doc': 'application/msword', + '.js': 'application/javascript', + '.xls': 'application/vnd.ms-excel', + '.ppt': 'application/vnd.ms-powerpoint', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation' +} as const; +export type ContentType = (typeof Mimes)[keyof typeof Mimes]; +export type ExtensionType = keyof typeof Mimes; + +export const defaultS3Options: { externalBaseURL?: string; maxFileSize?: number } & ClientOptions = + { + maxFileSize: 1024 ** 3, // 1GB + + useSSL: process.env.S3_USE_SSL === 'true', + endPoint: process.env.S3_ENDPOINT || 'localhost', + externalBaseURL: process.env.S3_EXTERNAL_BASE_URL, + accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', + secretKey: process.env.S3_SECRET_KEY || 'minioadmin', + port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, + transportAgent: process.env.HTTP_PROXY + ? new HttpProxyAgent(process.env.HTTP_PROXY) + : process.env.HTTPS_PROXY + ? new HttpsProxyAgent(process.env.HTTPS_PROXY) + : undefined + }; +export type S3Options = typeof defaultS3Options; + +export const S3Buckets = { + plugin: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', + public: process.env.S3_PUBLIC_BUCKET || 'fastgpt-public', + private: process.env.S3_PRIVATE_BUCKET || 'fastgpt-private' +} as const; +export type S3BucketName = (typeof S3Buckets)[keyof typeof S3Buckets]; + +export const S3SourcesSchema = z.enum([ + 'avatar', + 'chat', + 'dataset', + 'dataset-image', + 'invoice', + 'rawtext' +]); +export const S3Sources = S3SourcesSchema.enum; +export type S3SourceType = z.infer; + +export const CreateObjectKeyParamsSchema = z.object({ + filename: z.string().min(1), + source: S3SourcesSchema, + teamId: z.string().length(16) +}); +export type CreateObjectKeyParams = z.infer; + +export const CreatePostPresignedUrlParamsSchema = z.object({ + ...CreateObjectKeyParamsSchema.shape, + visibility: z.enum(['public', 'private']).default('private') +}); +export type CreatePostPresignedUrlParams = z.infer; + +export const CreatePostPresignedUrlResultSchema = z.object({ + url: z.string().min(1), + fields: z.record(z.string(), z.string()) +}); +export type CreatePostPresignedUrlResult = z.infer; diff --git a/packages/service/common/s3/usage-example.ts b/packages/service/common/s3/usage-example.ts new file mode 100644 index 000000000000..3addcb929090 --- /dev/null +++ b/packages/service/common/s3/usage-example.ts @@ -0,0 +1,84 @@ +/** + * S3 Sources 使用示例 + * 展示如何使用改进后的 S3 架构 + */ + +import { getS3AvatarSource } from './sources/avatar'; +import { getS3ChatSource } from './sources/chat'; + +// 使用示例 +export const s3UsageExample = async () => { + // 获取 avatar source（public bucket） + const avatarSource = getS3AvatarSource(); + + // 获取 chat source（private bucket） + const chatSource = getS3ChatSource(); + + // ✅ 现在可以直接调用 createPublicUrl 方法！ + const publicUrl = avatarSource.createPublicUrl('avatar/team123/2024_01_01/abc123_avatar.png'); + console.log('Avatar public URL:', publicUrl); + + // ✅ 也可以通过 getPublicBucket 获取 bucket 实例 + const publicBucket = avatarSource.getPublicBucket(); + const anotherPublicUrl = publicBucket.createPublicUrl( + 'avatar/team456/2024_01_01/def456_avatar.png' + ); + console.log('Another avatar public URL:', anotherPublicUrl); + + // ✅ 创建预签名 URL + const avatarPresignedUrl = await avatarSource.createPostPresignedUrl({ + filename: 'new-avatar.png', + teamId: '1234567890123456' + }); + console.log('Avatar presigned URL:', avatarPresignedUrl); + + // ✅ Private bucket 示例 + const chatPresignedUrl = await chatSource.createPostPresignedUrl({ + filename: 'chat-history.txt', + teamId: '1234567890123456' + }); + console.log('Chat presigned URL:', chatPresignedUrl); + + // ✅ 获取 private bucket 实例 + const privateBucket = chatSource.getPrivateBucket(); + console.log('Private bucket name:', privateBucket.name); +}; + +// 类型安全示例 +export const typeSafetyExample = () => { + const avatarSource = getS3AvatarSource(); + const chatSource = getS3ChatSource(); + + // ✅ TypeScript 现在知道 avatarSource.bucket 是 S3PublicBucket 类型 + // 所以可以安全地调用 createPublicUrl + const url1 = avatarSource.createPublicUrl('test.png'); + + // ✅ 也可以直接访问 bucket 的方法 + const url2 = avatarSource.bucket.createPublicUrl('test2.png'); + + // ❌ chatSource 没有 createPublicUrl 方法，因为它是 private bucket + // chatSource.createPublicUrl('test.png'); // TypeScript 错误 + + // ✅ 但 chatSource 可以访问基础的 bucket 方法 + console.log('Chat bucket name:', chatSource.bucketName); +}; + +// 单例验证示例 +export const singletonExample = () => { + const avatar1 = getS3AvatarSource(); + const avatar2 = getS3AvatarSource(); + + // 验证单例模式 + console.log('Same instance:', avatar1 === avatar2); // true + console.log('Same bucket:', avatar1.bucket === avatar2.bucket); // true + + const chat1 = getS3ChatSource(); + const chat2 = getS3ChatSource(); + + // 验证单例模式 + console.log('Same chat instance:', chat1 === chat2); // true + console.log('Same chat bucket:', chat1.bucket === chat2.bucket); // true + + // 不同类型的 source 使用不同的 bucket + console.log('Different buckets:', avatar1.bucket !== chat1.bucket); // true +}; diff --git a/packages/service/package.json b/packages/service/package.json index eb367ff8ed38..41438d10f7af 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -54,7 +54,8 @@ "tiktoken": "1.0.17", "tunnel": "^0.0.6", "turndown": "^7.1.2", - "winston": "^3.17.0" + "winston": "^3.17.0", + "zod": "^3.24.2" }, "devDependencies": { "@types/cookie": "^0.5.2", diff --git a/packages/web/i18n/en/account_info.json b/packages/web/i18n/en/account_info.json index 15a520732bef..c420e057d285 100644 --- a/packages/web/i18n/en/account_info.json +++ b/packages/web/i18n/en/account_info.json @@ -9,6 +9,8 @@ "app_amount": "App amount", "avatar": "Avatar", "avatar_selection_exception": "Abnormal avatar selection", + "avatar_can_only_select_one": "Avatar can only select one picture", + "avatar_can_only_select_jpg_png": "Avatar can only select jpg or png format", "balance": "balance", "billing_standard": "Standards", "cancel": "Cancel", diff --git a/packages/web/i18n/zh-CN/account_info.json b/packages/web/i18n/zh-CN/account_info.json index cd20076cdcc6..7822d8765995 100644 --- a/packages/web/i18n/zh-CN/account_info.json +++ b/packages/web/i18n/zh-CN/account_info.json @@ -9,6 +9,8 @@ "app_amount": "应用数量", "avatar": "头像", "avatar_selection_exception": "头像选择异常", + "avatar_can_only_select_one": "头像只能选择一张图片", + "avatar_can_only_select_jpg_png": "头像只能选择 jpg 或 png 格式", "balance": "余额", "billing_standard": "计费标准", "cancel": "取消", diff --git a/packages/web/i18n/zh-Hant/account_info.json b/packages/web/i18n/zh-Hant/account_info.json index 9c5560d48d91..5646df649802 100644 --- a/packages/web/i18n/zh-Hant/account_info.json +++ b/packages/web/i18n/zh-Hant/account_info.json @@ -9,6 +9,8 @@ "app_amount": "應用數量", "avatar": "頭像", "avatar_selection_exception": "頭像選擇異常", + "avatar_can_only_select_one": "頭像只能選擇一張圖片", + "avatar_can_only_select_jpg_png": "頭像只能選擇 jpg 或 png 格式", "balance": "餘額", "billing_standard": "計費標準", "cancel": "取消", diff --git a/projects/app/.env.template b/projects/app/.env.template index 308209973914..d9d383ebc66f 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -41,7 +41,10 @@ S3_PORT=9000 S3_USE_SSL=false S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin -S3_PLUGIN_BUCKET=fastgpt-plugin # 插件文件存储bucket +S3_PLUGIN_BUCKET=fastgpt-plugins # 插件文件存储bucket +S3_PUBLIC_BUCKET=fastgpt-public # 插件文件存储公开桶 +S3_PRIVATE_BUCKET=fastgpt-private # 插件文件存储公开桶 + # Redis URL REDIS_URL=redis://default:mypassword@127.0.0.1:6379 # mongo 数据库连接参数，本地开发连接远程数据库时，可能需要增加 directConnection=true 参数，才能连接上。 diff --git a/projects/app/src/pages/account/info/index.tsx b/projects/app/src/pages/account/info/index.tsx index 783cdfa85290..c08dab4d5bfd 100644 --- a/projects/app/src/pages/account/info/index.tsx +++ b/projects/app/src/pages/account/info/index.tsx @@ -46,6 +46,7 @@ import { getWorkorderURL } from '@/web/common/workorder/api'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import { useMount } from 'ahooks'; import MyDivider from '@fastgpt/web/components/common/MyDivider'; +import { useUploadAvatar } from '@/web/common/file/hooks/useUploadAvatar'; const RedeemCouponModal = dynamic(() => import('@/pageComponents/account/info/RedeemCouponModal'), { ssr: false @@ -140,14 +141,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { onClose: onCloseUpdateContact, onOpen: onOpenUpdateContact } = useDisclosure(); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false - }); const onclickSave = useCallback( async (data: UserType) => { @@ -164,6 +157,11 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { [reset, t, toast, updateUserInfo] ); + const { UploadAvatar, handleOpenSelectFile } = useUploadAvatar((avatar) => { + if (!userInfo) return; + onclickSave({ ...userInfo, avatar }); + }); + const labelStyles: BoxProps = { flex: '0 0 80px', color: 'var(--light-general-on-surface-lowest, var(--Gray-Modern-500, #667085))', @@ -241,6 +239,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} + {isPc ? ( {t('account_info:avatar')}  @@ -253,7 +252,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { border={theme.borders.base} overflow={'hidden'} boxShadow={'0 0 5px rgba(0,0,0,0.1)'} - onClick={onOpenSelectFile} + onClick={handleOpenSelectFile} > @@ -264,7 +263,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { flexDirection={'column'} alignItems={'center'} cursor={'pointer'} - onClick={onOpenSelectFile} + onClick={handleOpenSelectFile} > void }) => { )} + {feConfigs?.isPlus && ( {t('account_info:member_name')}  @@ -334,21 +334,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} {isOpenUpdatePsw && } {isOpenUpdateContact && } - - onSelectImage(e, { - maxW: 300, - maxH: 300, - callback: (src) => { - if (!userInfo) return; - onclickSave({ - ...userInfo, - avatar: src - }); - } - }) - } - /> ); }; diff --git a/projects/app/src/pages/api/support/user/account/update.ts b/projects/app/src/pages/api/support/user/account/update.ts index dfa451e29028..894772ed65bb 100644 --- a/projects/app/src/pages/api/support/user/account/update.ts +++ b/projects/app/src/pages/api/support/user/account/update.ts @@ -8,6 +8,7 @@ import { NextAPI } from '@/service/middleware/entry'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller'; import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; export type UserAccountUpdateQuery = {}; export type UserAccountUpdateBody = UserUpdateParams; @@ -37,6 +38,10 @@ async function handler( } // if avatar, update team member avatar if (avatar) { + if (tmb?.avatar) { + const objectKey = tmb.avatar.split('/').slice(4).join('/'); + await getS3AvatarSource().removeAvatar(objectKey); + } await MongoTeamMember.updateOne( { _id: tmbId @@ -45,7 +50,6 @@ async function handler( avatar } ).session(session); - await refreshSourceAvatar(avatar, tmb?.avatar, session); } }); diff --git a/projects/app/src/pages/api/support/user/account/updateAvatar.ts b/projects/app/src/pages/api/support/user/account/updateAvatar.ts new file mode 100644 index 000000000000..d199c3b7374b --- /dev/null +++ b/projects/app/src/pages/api/support/user/account/updateAvatar.ts @@ -0,0 +1,24 @@ +import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import { NextAPI } from '@/service/middleware/entry'; +import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/types'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; + +export type updateAvatarQuery = {}; + +export type updateAvatarBody = { + filename: string; +}; + +export type updateAvatarResponse = CreatePostPresignedUrlResult; + +async function handler( + req: ApiRequestProps, + _: ApiResponseType +): Promise { + const { filename } = req.body; + const { teamId } = await authCert({ req, authToken: true }); + return await getS3AvatarSource().createPostPresignedUrl({ teamId, filename }); +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/system/img/[...id].ts b/projects/app/src/pages/api/system/img/[...id].ts new file mode 100644 index 000000000000..90d0df6be51d --- /dev/null +++ b/projects/app/src/pages/api/system/img/[...id].ts @@ -0,0 +1,32 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import path from 'path'; + +import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; +import { Types } from '@fastgpt/service/common/mongo'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; + +// get the models available to the system +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { id } = req.query as { id: string[] }; + + const joined = id.join('/'); + const parsed = path.parse(joined); + const keys = path.format({ dir: parsed.dir, name: parsed.name, ext: '' }); + + if (Types.ObjectId.isValid(keys)) { + const { binary, mime } = await readMongoImg({ id: joined }); + res.setHeader('Content-Type', mime); + res.send(binary); + return; + } + + res.redirect(301, getS3AvatarSource().createPublicUrl(joined)); + } catch (error) { + jsonRes(res, { + code: 500, + error + }); + } +} diff --git a/projects/app/src/pages/api/system/img/[id].ts b/projects/app/src/pages/api/system/img/[id].ts deleted file mode 100644 index 882d21ab3c87..000000000000 --- a/projects/app/src/pages/api/system/img/[id].ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; - -import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; - -// get the models available to the system -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { id } = req.query as { id: string }; - - const { binary, mime } = await readMongoImg({ id }); - - res.setHeader('Content-Type', mime); - res.send(binary); - } catch (error) { - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/web/common/file/api.ts b/projects/app/src/web/common/file/api.ts index c73dad821095..79094366aecf 100644 --- a/projects/app/src/web/common/file/api.ts +++ b/projects/app/src/web/common/file/api.ts @@ -1,6 +1,6 @@ -import { DELETE, GET, POST } from '@/web/common/api/request'; +import { POST } from '@/web/common/api/request'; import type { UploadImgProps } from '@fastgpt/global/common/file/api.d'; -import type { UploadPresignedURLResponse } from '@fastgpt/service/common/s3/type'; +import type { CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/types'; import { type AxiosProgressEvent } from 'axios'; export const postUploadImg = (e: UploadImgProps) => POST('/common/file/uploadImage', e); @@ -32,3 +32,7 @@ export const postS3UploadFile = ( }, onUploadProgress }); + +export const getUploadAvatarPresignedUrl = (filename: string) => { + return POST('/support/user/account/updateAvatar', { filename }); +}; diff --git a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx b/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx new file mode 100644 index 000000000000..81f92d1c08ed --- /dev/null +++ b/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx @@ -0,0 +1,78 @@ +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; +import { base64ToFile, fileToBase64 } from '@/web/common/file/utils'; +import { compressBase64Img } from '@fastgpt/web/common/file/img'; +import { useToast } from '@fastgpt/web/hooks/useToast'; +import { useCallback, useRef, useState, useTransition } from 'react'; +import { useTranslation } from 'next-i18next'; + +export const useUploadAvatar = (onSuccess: (avatar: string) => void) => { + const { t } = useTranslation(); + const uploadAvatarRef = useRef(null); + const [isUploading, startUpload] = useTransition(); + const { toast } = useToast(); + const [_, setAvatar] = useState(); + + const handleOpenSelectFile = useCallback(() => { + if (!uploadAvatarRef.current) return; + uploadAvatarRef.current.click(); + }, []); + + const onUploadAvatarChange = useCallback( + async (e: React.ChangeEvent) => { + const files = e.target.files; + if (!files || files.length === 0) return; + if (files.length > 1) { + toast({ title: t('account_info:avatar_can_only_select_one'), status: 'warning' }); + return; + } + const file = files[0]; + if (!file) return; + + if (!file.name.match(/\.(jpg|png|jpeg)$/)) { + toast({ title: t('account_info:avatar_can_only_select_jpg_png'), status: 'warning' }); + return; + } + + startUpload(async () => { + const compressed = base64ToFile( + await compressBase64Img({ + base64Img: await fileToBase64(file), + maxW: 300, + maxH: 300 + }), + file.name + ); + const { url, fields } = await getUploadAvatarPresignedUrl(file.name); + const formData = new FormData(); + Object.entries(fields).forEach(([k, v]) => formData.set(k, v)); + formData.set('file', compressed); + await fetch(url, { method: 'POST', body: formData }); // 204 + + const prefix = '/api/system/img/'; + const avatar = `${prefix}${fields.key}`; + setAvatar(avatar); + onSuccess(avatar); + }); + }, + [toast, onSuccess, t] + ); + + const UploadAvatar = () => { + return ( + + ); + }; + + return { + isUploading, + UploadAvatar, + handleOpenSelectFile + }; +}; diff --git a/projects/app/src/web/common/file/utils.ts b/projects/app/src/web/common/file/utils.ts index 160ad29ea3ac..38358902d874 100644 --- a/projects/app/src/web/common/file/utils.ts +++ b/projects/app/src/web/common/file/utils.ts @@ -32,3 +32,15 @@ export const fileToBase64 = (file: File) => { reader.onerror = (error) => reject(error); }); }; + +export const base64ToFile = (base64: string, filename: string) => { + const arr = base64.split(','); + const mime = arr[0].match(/:(.*?);/)?.[1]; + const bstr = atob(arr[1]); + let n = bstr.length; + const u8arr = new Uint8Array(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n); + } + return new File([u8arr], filename, { type: mime }); +}; From 9b7ddc1d88f042102d444a7d543d1024fcf8752b Mon Sep 17 00:00:00 2001 From: xqvvu Date: Fri, 26 Sep 2025 16:15:50 +0800 Subject: [PATCH 2/4] feat: migrate apps' and dataset's avatar to minio --- packages/service/common/s3/buckets/base.ts | 53 +++++++----- packages/service/common/s3/buckets/public.ts | 33 +++++--- packages/service/common/s3/helpers.ts | 8 ++ packages/service/common/s3/interface.ts | 15 ++-- packages/service/common/s3/lifecycle.ts | 23 +++++ packages/service/common/s3/sources/avatar.ts | 32 +++++-- packages/service/common/s3/types.ts | 13 ++- packages/service/common/s3/usage-example.ts | 84 ------------------- .../common/Modal/EditResourceModal.tsx | 32 ++++--- projects/app/src/pages/account/info/index.tsx | 18 ++-- .../updateAvatar.ts => common/file/avatar.ts} | 5 +- projects/app/src/pages/api/core/app/update.ts | 13 ++- .../app/src/pages/api/core/dataset/update.ts | 13 ++- .../pages/api/support/user/account/update.ts | 4 +- projects/app/src/web/common/file/api.ts | 4 +- .../web/common/file/hooks/useUploadAvatar.tsx | 51 ++++++----- 16 files changed, 218 insertions(+), 183 deletions(-) create mode 100644 packages/service/common/s3/lifecycle.ts delete mode 100644 packages/service/common/s3/usage-example.ts rename projects/app/src/pages/api/{support/user/account/updateAvatar.ts => common/file/avatar.ts} (90%) diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts index e2b269aeea03..5a9d8c7d248b 100644 --- a/packages/service/common/s3/buckets/base.ts +++ b/packages/service/common/s3/buckets/base.ts @@ -1,13 +1,19 @@ -import { Client } from 'minio'; +import { Client, type RemoveOptions, type CopyConditions, type LifecycleConfig } from 'minio'; import { defaultS3Options, + type CreatePostPresignedUrlOptions, type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3BucketName, type S3Options } from '../types'; import type { IBucketBasicOperations } from '../interface'; -import { createObjectKey, createPresignedUrlExpires, inferContentType } from '../helpers'; +import { + createObjectKey, + createPresignedUrlExpires, + createTempObjectKey, + inferContentType +} from '../helpers'; export class S3BaseBucket implements IBucketBasicOperations { public client: Client; @@ -16,11 +22,11 @@ export class S3BaseBucket implements IBucketBasicOperations { * * @param _bucket the bucket you want to operate * @param options the options for the s3 client - * @param afterInit the function to be called after instantiating the s3 service + * @param afterInits the function to be called after instantiating the s3 service */ constructor( private readonly _bucket: S3BucketName, - private readonly afterInit?: () => Promise | void, + private readonly afterInits?: (() => Promise | void)[], public options: Partial = defaultS3Options ) { options = { ...defaultS3Options, ...options }; @@ -31,45 +37,54 @@ export class S3BaseBucket implements IBucketBasicOperations { if (!(await this.exist())) { await this.client.makeBucket(this._bucket); } - await this.afterInit?.(); + await Promise.all(this.afterInits?.map((afterInit) => afterInit()) ?? []); }; init(); } - async exist(): Promise { - return await this.client.bucketExists(this._bucket); - } - get name(): string { return this._bucket; } - upload(): Promise { - throw new Error('Method not implemented.'); + async move(src: string, dst: string, options?: CopyConditions): Promise { + const bucket = this.name; + await this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options); + return this.client.removeObject(bucket, src); } - download(): Promise { - throw new Error('Method not implemented.'); + copy(src: string, dst: string, options?: CopyConditions): ReturnType { + return this.client.copyObject(this.name, src, dst, options); } - delete(objectKey: string): Promise { - return this.client.removeObject(this._bucket, objectKey); + exist(): Promise { + return this.client.bucketExists(this.name); + } + + delete(objectKey: string, options?: RemoveOptions): Promise { + return this.client.removeObject(this.name, objectKey, options); } get(): Promise { throw new Error('Method not implemented.'); } + lifecycle(): Promise { + return this.client.getBucketLifecycle(this.name); + } + async createPostPresignedUrl( - params: CreatePostPresignedUrlParams + params: CreatePostPresignedUrlParams, + options: CreatePostPresignedUrlOptions = {} ): Promise { - const maxFileSize = this.options.maxFileSize as number; + const { temporay } = options; const contentType = inferContentType(params.filename); + const maxFileSize = this.options.maxFileSize as number; + const key = temporay ? createTempObjectKey(params) : createObjectKey(params); const policy = this.client.newPostPolicy(); - policy.setBucket(this._bucket); + policy.setKey(key); + policy.setBucket(this.name); policy.setContentType(contentType); - policy.setKey(createObjectKey(params)); policy.setContentLengthRange(1, maxFileSize); policy.setExpires(createPresignedUrlExpires(10)); policy.setUserMetaData({ diff --git a/packages/service/common/s3/buckets/public.ts b/packages/service/common/s3/buckets/public.ts index bd592abf9fe1..4843c910dc94 100644 --- a/packages/service/common/s3/buckets/public.ts +++ b/packages/service/common/s3/buckets/public.ts @@ -2,26 +2,36 @@ import { S3BaseBucket } from './base'; import { createBucketPolicy } from '../helpers'; import { S3Buckets, + type CreatePostPresignedUrlOptions, type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options } from '../types'; import type { IPublicBucketOperations } from '../interface'; +import { lifecycleOfTemporaryAvatars } from '../lifecycle'; export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperations { constructor(options?: Partial) { super( S3Buckets.public, - async () => { - const bucket = this.name; - const policy = createBucketPolicy(bucket); - try { - await this.client.setBucketPolicy(bucket, policy); - } catch (error) { - // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error, - // maybe we can ignore the error, or we have other plan to handle this. + [ + // set bucket policy + async () => { + const bucket = this.name; + const policy = createBucketPolicy(bucket); + try { + await this.client.setBucketPolicy(bucket, policy); + } catch (error) { + // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error, + // maybe we can ignore the error, or we have other plan to handle this. + } + }, + // set bucket lifecycle + async () => { + const bucket = this.name; + await this.client.setBucketLifecycle(bucket, lifecycleOfTemporaryAvatars); } - }, + ], options ); } @@ -36,8 +46,9 @@ export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperati } override createPostPresignedUrl( - params: Omit + params: Omit, + options: CreatePostPresignedUrlOptions = {} ): Promise { - return super.createPostPresignedUrl({ ...params, visibility: 'public' }); + return super.createPostPresignedUrl({ ...params, visibility: 'public' }, options); } } diff --git a/packages/service/common/s3/helpers.ts b/packages/service/common/s3/helpers.ts index c8b5c30f3d98..2b230a3f5342 100644 --- a/packages/service/common/s3/helpers.ts +++ b/packages/service/common/s3/helpers.ts @@ -69,3 +69,11 @@ export function createObjectKey({ source, teamId, filename }: CreateObjectKeyPar const id = crypto.randomBytes(16).toString('hex'); return `${source}/${teamId}/${date}/${id}_${filename}`; } + +/** + * create temporary s3 object key by source, team ID and filename + */ +export function createTempObjectKey(params: CreateObjectKeyParams): string { + const origin = createObjectKey(params); + return `temp/${origin}`; +} diff --git a/packages/service/common/s3/interface.ts b/packages/service/common/s3/interface.ts index 79517eaedfb1..3f99a6fb4c92 100644 --- a/packages/service/common/s3/interface.ts +++ b/packages/service/common/s3/interface.ts @@ -1,14 +1,17 @@ -import type { CreateObjectKeyParams } from './types'; +import { type LifecycleConfig, type Client, type CopyConditions, type RemoveOptions } from 'minio'; +import type { CreateObjectKeyParams, CreatePostPresignedUrlOptions } from './types'; export interface IBucketBasicOperations { get name(): string; - exist(): Promise; - upload(): Promise; - download(): Promise; - delete(objectKey: string): Promise; get(): Promise; + exist(): Promise; + delete(objectKey: string, options?: RemoveOptions): Promise; + move(src: string, dst: string, options?: CopyConditions): Promise; + copy(src: string, dst: string, options?: CopyConditions): ReturnType; + lifecycle(): Promise; createPostPresignedUrl( - params: CreateObjectKeyParams + params: CreateObjectKeyParams, + options?: CreatePostPresignedUrlOptions ): Promise<{ url: string; fields: Record }>; } diff --git a/packages/service/common/s3/lifecycle.ts b/packages/service/common/s3/lifecycle.ts new file mode 100644 index 000000000000..12c28c1ca746 --- /dev/null +++ b/packages/service/common/s3/lifecycle.ts @@ -0,0 +1,23 @@ +import { type LifecycleRule, type LifecycleConfig } from 'minio'; + +export function createLifeCycleConfig(rule: LifecycleRule): LifecycleConfig { + return { + Rule: [rule] + }; +} + +export function assembleLifeCycleConfigs(...configs: LifecycleConfig[]): LifecycleConfig { + if (configs.length === 0) return { Rule: [] }; + return { + Rule: configs.flatMap((config) => config.Rule) + }; +} + +export const lifecycleOfTemporaryAvatars = createLifeCycleConfig({ + ID: 'Temporary Avatars Rule', + Prefix: 'temp/avatar/', + Status: 'Enabled', + Expiration: { + Days: 1 + } +}); diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts index 97f7666a012c..f71d29e472c9 100644 --- a/packages/service/common/s3/sources/avatar.ts +++ b/packages/service/common/s3/sources/avatar.ts @@ -1,6 +1,8 @@ import { S3BaseSource } from './base'; import { S3Sources, + S3APIPrefix, + type CreatePostPresignedUrlOptions, type CreatePostPresignedUrlParams, type CreatePostPresignedUrlResult, type S3Options @@ -17,20 +19,36 @@ class S3AvatarSource extends S3BaseSource { } override createPostPresignedUrl( - params: Omit + params: Omit, + options: CreatePostPresignedUrlOptions = {} ): Promise { - return this.bucket.createPostPresignedUrl({ - ...params, - source: S3Sources.avatar - }); + return this.bucket.createPostPresignedUrl( + { + ...params, + source: S3Sources.avatar + }, + options + ); } createPublicUrl(objectKey: string): string { return this.bucket.createPublicUrl(objectKey); } - removeAvatar(objectKey: string): Promise { - return this.bucket.delete(objectKey); + createAvatarObjectKey(avatarWithPrefix: string): string { + return avatarWithPrefix.replace(S3APIPrefix.avatar, ''); + } + + removeAvatar(avatarWithPrefix: string): Promise { + const avatarObjectKey = this.createAvatarObjectKey(avatarWithPrefix); + return this.bucket.delete(avatarObjectKey); + } + + async moveAvatarFromTemp(tempAvatarWithPrefix: string): Promise { + const tempAvatarObjectKey = this.createAvatarObjectKey(tempAvatarWithPrefix); + const avatarObjectKey = tempAvatarObjectKey.replace(`${S3Sources.temp}/`, ''); + await this.bucket.move(tempAvatarObjectKey, avatarObjectKey); + return S3APIPrefix.avatar + avatarObjectKey; } } diff --git a/packages/service/common/s3/types.ts b/packages/service/common/s3/types.ts index b7e66f74bff2..d2be9c357448 100644 --- a/packages/service/common/s3/types.ts +++ b/packages/service/common/s3/types.ts @@ -69,7 +69,8 @@ export const S3SourcesSchema = z.enum([ 'dataset', 'dataset-image', 'invoice', - 'rawtext' + 'rawtext', + 'temp' ]); export const S3Sources = S3SourcesSchema.enum; export type S3SourceType = z.infer; @@ -87,8 +88,18 @@ export const CreatePostPresignedUrlParamsSchema = z.object({ }); export type CreatePostPresignedUrlParams = z.infer; +export const CreatePostPresignedUrlOptionsSchema = z.object({ + temporay: z.boolean().optional() +}); +export type CreatePostPresignedUrlOptions = z.infer; + export const CreatePostPresignedUrlResultSchema = z.object({ url: z.string().min(1), fields: z.record(z.string(), z.string()) }); export type CreatePostPresignedUrlResult = z.infer; + +export const S3APIPrefix = { + avatar: '/api/system/img/' +} as const; +export type S3APIPrefixType = (typeof S3APIPrefix)[keyof typeof S3APIPrefix]; diff --git a/packages/service/common/s3/usage-example.ts b/packages/service/common/s3/usage-example.ts deleted file mode 100644 index 3addcb929090..000000000000 --- a/packages/service/common/s3/usage-example.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * S3 Sources 使用示例 - * 展示如何使用改进后的 S3 架构 - */ - -import { getS3AvatarSource } from './sources/avatar'; -import { getS3ChatSource } from './sources/chat'; - -// 使用示例 -export const s3UsageExample = async () => { - // 获取 avatar source（public bucket） - const avatarSource = getS3AvatarSource(); - - // 获取 chat source（private bucket） - const chatSource = getS3ChatSource(); - - // ✅ 现在可以直接调用 createPublicUrl 方法！ - const publicUrl = avatarSource.createPublicUrl('avatar/team123/2024_01_01/abc123_avatar.png'); - console.log('Avatar public URL:', publicUrl); - - // ✅ 也可以通过 getPublicBucket 获取 bucket 实例 - const publicBucket = avatarSource.getPublicBucket(); - const anotherPublicUrl = publicBucket.createPublicUrl( - 'avatar/team456/2024_01_01/def456_avatar.png' - ); - console.log('Another avatar public URL:', anotherPublicUrl); - - // ✅ 创建预签名 URL - const avatarPresignedUrl = await avatarSource.createPostPresignedUrl({ - filename: 'new-avatar.png', - teamId: '1234567890123456' - }); - console.log('Avatar presigned URL:', avatarPresignedUrl); - - // ✅ Private bucket 示例 - const chatPresignedUrl = await chatSource.createPostPresignedUrl({ - filename: 'chat-history.txt', - teamId: '1234567890123456' - }); - console.log('Chat presigned URL:', chatPresignedUrl); - - // ✅ 获取 private bucket 实例 - const privateBucket = chatSource.getPrivateBucket(); - console.log('Private bucket name:', privateBucket.name); -}; - -// 类型安全示例 -export const typeSafetyExample = () => { - const avatarSource = getS3AvatarSource(); - const chatSource = getS3ChatSource(); - - // ✅ TypeScript 现在知道 avatarSource.bucket 是 S3PublicBucket 类型 - // 所以可以安全地调用 createPublicUrl - const url1 = avatarSource.createPublicUrl('test.png'); - - // ✅ 也可以直接访问 bucket 的方法 - const url2 = avatarSource.bucket.createPublicUrl('test2.png'); - - // ❌ chatSource 没有 createPublicUrl 方法，因为它是 private bucket - // chatSource.createPublicUrl('test.png'); // TypeScript 错误 - - // ✅ 但 chatSource 可以访问基础的 bucket 方法 - console.log('Chat bucket name:', chatSource.bucketName); -}; - -// 单例验证示例 -export const singletonExample = () => { - const avatar1 = getS3AvatarSource(); - const avatar2 = getS3AvatarSource(); - - // 验证单例模式 - console.log('Same instance:', avatar1 === avatar2); // true - console.log('Same bucket:', avatar1.bucket === avatar2.bucket); // true - - const chat1 = getS3ChatSource(); - const chat2 = getS3ChatSource(); - - // 验证单例模式 - console.log('Same chat instance:', chat1 === chat2); // true - console.log('Same chat bucket:', chat1.bucket === chat2.bucket); // true - - // 不同类型的 source 使用不同的 bucket - console.log('Different buckets:', avatar1.bucket !== chat1.bucket); // true -}; diff --git a/projects/app/src/components/common/Modal/EditResourceModal.tsx b/projects/app/src/components/common/Modal/EditResourceModal.tsx index 85c8d06d2c49..5d2114eb2a8a 100644 --- a/projects/app/src/components/common/Modal/EditResourceModal.tsx +++ b/projects/app/src/components/common/Modal/EditResourceModal.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { ModalFooter, ModalBody, Input, Button, Box, Textarea, HStack } from '@chakra-ui/react'; import MyModal from '@fastgpt/web/components/common/MyModal/index'; import { useTranslation } from 'next-i18next'; @@ -8,6 +8,7 @@ import { useForm } from 'react-hook-form'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import Avatar from '@fastgpt/web/components/common/Avatar'; +import { useUploadAvatar } from '@/web/common/file/hooks/useUploadAvatar'; export type EditResourceInfoFormType = { id: string; @@ -41,13 +42,16 @@ const EditResourceModal = ({ } ); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false + const afterUploadAvatar = useCallback( + (avatar: string) => { + console.log('avatar', avatar); + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar({ + temporay: true, + onSuccess: afterUploadAvatar }); return ( @@ -64,7 +68,7 @@ const EditResourceModal = ({ h={'2rem'} cursor={'pointer'} borderRadius={'sm'} - onClick={onOpenSelectFile} + onClick={handleFileSelectorOpen} /> - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> + ); }; diff --git a/projects/app/src/pages/account/info/index.tsx b/projects/app/src/pages/account/info/index.tsx index c08dab4d5bfd..1df7947ad238 100644 --- a/projects/app/src/pages/account/info/index.tsx +++ b/projects/app/src/pages/account/info/index.tsx @@ -157,9 +157,15 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { [reset, t, toast, updateUserInfo] ); - const { UploadAvatar, handleOpenSelectFile } = useUploadAvatar((avatar) => { - if (!userInfo) return; - onclickSave({ ...userInfo, avatar }); + const afterUploadAvatar = useCallback( + (avatar: string) => { + if (!userInfo) return; + onclickSave({ ...userInfo, avatar }); + }, + [onclickSave, userInfo] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar({ + onSuccess: afterUploadAvatar }); const labelStyles: BoxProps = { @@ -239,7 +245,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} - + {isPc ? ( {t('account_info:avatar')}  @@ -252,7 +258,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { border={theme.borders.base} overflow={'hidden'} boxShadow={'0 0 5px rgba(0,0,0,0.1)'} - onClick={handleOpenSelectFile} + onClick={handleFileSelectorOpen} > @@ -263,7 +269,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { flexDirection={'column'} alignItems={'center'} cursor={'pointer'} - onClick={handleOpenSelectFile} + onClick={handleFileSelectorOpen} > , _: ApiResponseType ): Promise { - const { filename } = req.body; + const { filename, temporay } = req.body; const { teamId } = await authCert({ req, authToken: true }); - return await getS3AvatarSource().createPostPresignedUrl({ teamId, filename }); + return await getS3AvatarSource().createPostPresignedUrl({ teamId, filename }, { temporay }); } export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/app/update.ts b/projects/app/src/pages/api/core/app/update.ts index 5709f8ccc7f9..0ab2cd802092 100644 --- a/projects/app/src/pages/api/core/app/update.ts +++ b/projects/app/src/pages/api/core/app/update.ts @@ -27,6 +27,7 @@ import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nAppType } from '@fastgpt/service/support/user/audit/util'; import { i18nT } from '@fastgpt/web/i18n/utils'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; export type AppUpdateQuery = { appId: string; @@ -111,13 +112,19 @@ async function handler(req: ApiRequestProps) { nodes }); - await refreshSourceAvatar(avatar, app.avatar, session); + const permanentAvatar = await (async () => { + if (avatar) { + const s3AvatarSource = getS3AvatarSource(); + await s3AvatarSource.removeAvatar(app.avatar); // 移除旧的头像 + return await s3AvatarSource.moveAvatarFromTemp(avatar); // 永久化临时的预览头像 + } + })(); if (app.type === AppTypeEnum.toolSet && avatar) { await MongoApp.updateMany( { parentId: appId, teamId: app.teamId }, { - avatar + avatar: permanentAvatar }, { session } ); @@ -129,7 +136,7 @@ async function handler(req: ApiRequestProps) { ...parseParentIdInMongo(parentId), ...(name && { name }), ...(type && { type }), - ...(avatar && { avatar }), + ...(avatar && { avatar: permanentAvatar }), ...(intro !== undefined && { intro }), ...(teamTags && { teamTags }), ...(nodes && { diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts index a2b0611ad353..5cd9ad7d51e7 100644 --- a/projects/app/src/pages/api/core/dataset/update.ts +++ b/projects/app/src/pages/api/core/dataset/update.ts @@ -39,6 +39,7 @@ import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model'; import { computedCollectionChunkSettings } from '@fastgpt/global/core/dataset/training/utils'; import { getResourceOwnedClbs } from '@fastgpt/service/support/permission/controller'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; export type DatasetUpdateQuery = {}; export type DatasetUpdateResponse = any; @@ -202,12 +203,20 @@ async function handler( return flattenObjectWithConditions(apiDatasetServer); })(); + const permanentAvatar = await (async () => { + if (avatar) { + const s3AvatarSource = getS3AvatarSource(); + await s3AvatarSource.removeAvatar(dataset.avatar); // 移除旧的头像 + return await s3AvatarSource.moveAvatarFromTemp(avatar); // 永久化临时的预览头像 + } + })(); + await MongoDataset.findByIdAndUpdate( id, { ...parseParentIdInMongo(parentId), ...(name && { name }), - ...(avatar && { avatar }), + ...(avatar && { avatar: permanentAvatar }), ...(agentModel && { agentModel }), ...(vlmModel && { vlmModel }), ...(websiteConfig && { websiteConfig }), @@ -224,8 +233,6 @@ async function handler( dataset, autoSync }); - - await refreshSourceAvatar(avatar, dataset.avatar, session); }; await mongoSessionRun(async (session) => { diff --git a/projects/app/src/pages/api/support/user/account/update.ts b/projects/app/src/pages/api/support/user/account/update.ts index 894772ed65bb..89da6a801f1e 100644 --- a/projects/app/src/pages/api/support/user/account/update.ts +++ b/projects/app/src/pages/api/support/user/account/update.ts @@ -39,9 +39,9 @@ async function handler( // if avatar, update team member avatar if (avatar) { if (tmb?.avatar) { - const objectKey = tmb.avatar.split('/').slice(4).join('/'); - await getS3AvatarSource().removeAvatar(objectKey); + await getS3AvatarSource().removeAvatar(tmb.avatar); } + await MongoTeamMember.updateOne( { _id: tmbId diff --git a/projects/app/src/web/common/file/api.ts b/projects/app/src/web/common/file/api.ts index 79094366aecf..60bed8f0014f 100644 --- a/projects/app/src/web/common/file/api.ts +++ b/projects/app/src/web/common/file/api.ts @@ -33,6 +33,6 @@ export const postS3UploadFile = ( onUploadProgress }); -export const getUploadAvatarPresignedUrl = (filename: string) => { - return POST('/support/user/account/updateAvatar', { filename }); +export const getUploadAvatarPresignedUrl = (params: { filename: string; temporay: boolean }) => { + return POST('/common/file/avatar', params); }; diff --git a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx b/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx index 81f92d1c08ed..fa90cbe32977 100644 --- a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx +++ b/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx @@ -2,17 +2,22 @@ import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; import { base64ToFile, fileToBase64 } from '@/web/common/file/utils'; import { compressBase64Img } from '@fastgpt/web/common/file/img'; import { useToast } from '@fastgpt/web/hooks/useToast'; -import { useCallback, useRef, useState, useTransition } from 'react'; +import { useCallback, useRef, useTransition } from 'react'; import { useTranslation } from 'next-i18next'; -export const useUploadAvatar = (onSuccess: (avatar: string) => void) => { +export const useUploadAvatar = ({ + temporay = false, + onSuccess +}: { + temporay?: boolean; + onSuccess: (avatar: string) => void; +}) => { + const { toast } = useToast(); const { t } = useTranslation(); + const [uploading, startUpload] = useTransition(); const uploadAvatarRef = useRef(null); - const [isUploading, startUpload] = useTransition(); - const { toast } = useToast(); - const [_, setAvatar] = useState(); - const handleOpenSelectFile = useCallback(() => { + const handleFileSelectorOpen = useCallback(() => { if (!uploadAvatarRef.current) return; uploadAvatarRef.current.click(); }, []); @@ -20,16 +25,21 @@ export const useUploadAvatar = (onSuccess: (avatar: string) => void) => { const onUploadAvatarChange = useCallback( async (e: React.ChangeEvent) => { const files = e.target.files; - if (!files || files.length === 0) return; + + if (!files || files.length === 0) { + e.target.value = ''; + return; + } if (files.length > 1) { toast({ title: t('account_info:avatar_can_only_select_one'), status: 'warning' }); + e.target.value = ''; return; } - const file = files[0]; - if (!file) return; + const file = files[0]!; if (!file.name.match(/\.(jpg|png|jpeg)$/)) { toast({ title: t('account_info:avatar_can_only_select_jpg_png'), status: 'warning' }); + e.target.value = ''; return; } @@ -42,22 +52,25 @@ export const useUploadAvatar = (onSuccess: (avatar: string) => void) => { }), file.name ); - const { url, fields } = await getUploadAvatarPresignedUrl(file.name); + const { url, fields } = await getUploadAvatarPresignedUrl({ + filename: file.name, + temporay + }); const formData = new FormData(); Object.entries(fields).forEach(([k, v]) => formData.set(k, v)); formData.set('file', compressed); await fetch(url, { method: 'POST', body: formData }); // 204 - const prefix = '/api/system/img/'; - const avatar = `${prefix}${fields.key}`; - setAvatar(avatar); + const avatar = `/api/system/img/${fields.key}`; onSuccess(avatar); + + e.target.value = ''; }); }, - [toast, onSuccess, t] + [t, temporay, toast, onSuccess] ); - const UploadAvatar = () => { + const Component = useCallback(() => { return ( void) => { onChange={onUploadAvatarChange} /> ); - }; + }, [onUploadAvatarChange]); return { - isUploading, - UploadAvatar, - handleOpenSelectFile + uploading, + Component, + handleFileSelectorOpen }; }; From a630d53db130dc70d9711f6c1e0cd01a848eb160 Mon Sep 17 00:00:00 2001 From: xqvvu Date: Mon, 29 Sep 2025 15:31:06 +0800 Subject: [PATCH 3/4] feat: migrate more avatars to minio --- .../web/common/file/hooks/useUploadAvatar.tsx | 34 ++++++++-------- packages/web/common/file/utils.ts | 21 ++++++++++ .../common/Modal/EditResourceModal.tsx | 15 ++++--- .../account/team/EditInfoModal.tsx | 39 +++++++++---------- projects/app/src/pages/account/info/index.tsx | 12 ++++-- 5 files changed, 73 insertions(+), 48 deletions(-) rename {projects/app/src => packages}/web/common/file/hooks/useUploadAvatar.tsx (75%) diff --git a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx b/packages/web/common/file/hooks/useUploadAvatar.tsx similarity index 75% rename from projects/app/src/web/common/file/hooks/useUploadAvatar.tsx rename to packages/web/common/file/hooks/useUploadAvatar.tsx index fa90cbe32977..10bf1ef04897 100644 --- a/projects/app/src/web/common/file/hooks/useUploadAvatar.tsx +++ b/packages/web/common/file/hooks/useUploadAvatar.tsx @@ -1,17 +1,20 @@ -import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; -import { base64ToFile, fileToBase64 } from '@/web/common/file/utils'; -import { compressBase64Img } from '@fastgpt/web/common/file/img'; -import { useToast } from '@fastgpt/web/hooks/useToast'; +import { base64ToFile, fileToBase64 } from '../utils'; +import { compressBase64Img } from '../img'; +import { useToast } from '../../../hooks/useToast'; import { useCallback, useRef, useTransition } from 'react'; import { useTranslation } from 'next-i18next'; +import { type CreatePostPresignedUrlResult } from '../../../../service/common/s3/types'; -export const useUploadAvatar = ({ - temporay = false, - onSuccess -}: { - temporay?: boolean; - onSuccess: (avatar: string) => void; -}) => { +export const useUploadAvatar = ( + api: (params: { filename: string; temporay: boolean }) => Promise, + { + temporay = false, + onSuccess + }: { + temporay?: boolean; + onSuccess?: (avatar: string) => void; + } = {} +) => { const { toast } = useToast(); const { t } = useTranslation(); const [uploading, startUpload] = useTransition(); @@ -52,22 +55,19 @@ export const useUploadAvatar = ({ }), file.name ); - const { url, fields } = await getUploadAvatarPresignedUrl({ - filename: file.name, - temporay - }); + const { url, fields } = await api({ filename: file.name, temporay }); const formData = new FormData(); Object.entries(fields).forEach(([k, v]) => formData.set(k, v)); formData.set('file', compressed); await fetch(url, { method: 'POST', body: formData }); // 204 const avatar = `/api/system/img/${fields.key}`; - onSuccess(avatar); + onSuccess?.(avatar); e.target.value = ''; }); }, - [t, temporay, toast, onSuccess] + [t, temporay, toast, onSuccess, api] ); const Component = useCallback(() => { diff --git a/packages/web/common/file/utils.ts b/packages/web/common/file/utils.ts index 7fd10744530f..f4107301d585 100644 --- a/packages/web/common/file/utils.ts +++ b/packages/web/common/file/utils.ts @@ -99,3 +99,24 @@ async function detectFileEncoding(file: File): Promise { return encoding || 'utf-8'; } + +export const fileToBase64 = (file: File) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => resolve(reader.result as string); + reader.onerror = (error) => reject(error); + }); +}; + +export const base64ToFile = (base64: string, filename: string) => { + const arr = base64.split(','); + const mime = arr[0].match(/:(.*?);/)?.[1]; + const bstr = atob(arr[1]); + let n = bstr.length; + const u8arr = new Uint8Array(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n); + } + return new File([u8arr], filename, { type: mime }); +}; diff --git a/projects/app/src/components/common/Modal/EditResourceModal.tsx b/projects/app/src/components/common/Modal/EditResourceModal.tsx index 5d2114eb2a8a..9827e2bdadc1 100644 --- a/projects/app/src/components/common/Modal/EditResourceModal.tsx +++ b/projects/app/src/components/common/Modal/EditResourceModal.tsx @@ -5,10 +5,10 @@ import { useTranslation } from 'next-i18next'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; import { useForm } from 'react-hook-form'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import Avatar from '@fastgpt/web/components/common/Avatar'; -import { useUploadAvatar } from '@/web/common/file/hooks/useUploadAvatar'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; export type EditResourceInfoFormType = { id: string; @@ -49,10 +49,13 @@ const EditResourceModal = ({ }, [setValue] ); - const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar({ - temporay: true, - onSuccess: afterUploadAvatar - }); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + temporay: true, + onSuccess: afterUploadAvatar + } + ); return ( diff --git a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx index db6f4574b403..0e0a0e95a114 100644 --- a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx +++ b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx @@ -1,7 +1,6 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { useForm } from 'react-hook-form'; import { useTranslation } from 'next-i18next'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { @@ -21,6 +20,8 @@ import { type CreateTeamProps } from '@fastgpt/global/support/user/team/controll import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants'; import Icon from '@fastgpt/web/components/common/Icon'; import dynamic from 'next/dynamic'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; const UpdateContact = dynamic(() => import('@/components/support/user/inform/UpdateContactModal')); export type EditTeamFormDataType = CreateTeamProps & { @@ -50,15 +51,6 @@ function EditModal({ const avatar = watch('avatar'); const notificationAccount = watch('notificationAccount'); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png,.svg', - multiple: false - }); - const { mutate: onclickCreate, isLoading: creating } = useRequest({ mutationFn: async (data: CreateTeamProps) => { return postCreateTeam(data); @@ -88,6 +80,19 @@ function EditModal({ const { isOpen: isOpenContact, onClose: onCloseContact, onOpen: onOpenContact } = useDisclosure(); + const afterUploadAvatar = useCallback( + (avatar: string) => { + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); + return ( {t('account_team:set_name_avatar')} + )} - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> {isOpenContact && ( import('@/pageComponents/account/info/RedeemCouponModal'), { ssr: false @@ -164,9 +165,12 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { }, [onclickSave, userInfo] ); - const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar({ - onSuccess: afterUploadAvatar - }); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); const labelStyles: BoxProps = { flex: '0 0 80px', From 77157bcdc0ccf5cc481f71ee1e4be8d582b072b7 Mon Sep 17 00:00:00 2001 From: xqvvu Date: Mon, 29 Sep 2025 15:37:01 +0800 Subject: [PATCH 4/4] fix: lock file --- pnpm-lock.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7da780021d45..066581a4aea0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -282,6 +282,9 @@ importers: winston: specifier: ^3.17.0 version: 3.17.0 + zod: + specifier: ^3.24.2 + version: 3.25.51 devDependencies: '@types/cookie': specifier: ^0.5.2 @@ -12231,8 +12234,8 @@ snapshots: express-rate-limit: 7.5.0(express@5.1.0) pkce-challenge: 5.0.0 raw-body: 3.0.0 - zod: 3.24.2 - zod-to-json-schema: 3.24.5(zod@3.24.2) + zod: 3.25.51 + zod-to-json-schema: 3.24.5(zod@3.25.51) transitivePeerDependencies: - supports-color @@ -21900,9 +21903,9 @@ snapshots: - terser - typescript - zod-to-json-schema@3.24.5(zod@3.24.2): + zod-to-json-schema@3.24.5(zod@3.25.51): dependencies: - zod: 3.24.2 + zod: 3.25.51 zod-validation-error@3.4.0(zod@3.25.51): dependencies: