diff --git a/packages/global/common/file/minioTtl/type.d.ts b/packages/global/common/file/minioTtl/type.d.ts new file mode 100644 index 000000000000..f9442368d0ca --- /dev/null +++ b/packages/global/common/file/minioTtl/type.d.ts @@ -0,0 +1,6 @@ +export type S3TtlSchemaType = { + _id: string; + bucketName: string; + minioKey: string; + expiredTime: Date; +}; diff --git a/packages/service/common/file/s3Ttl/controller.ts b/packages/service/common/file/s3Ttl/controller.ts new file mode 100644 index 000000000000..c0cea060b2c6 --- /dev/null +++ b/packages/service/common/file/s3Ttl/controller.ts @@ -0,0 +1,65 @@ +import { MongoS3TTL } from './schema'; +import { S3BucketManager } from '../../s3/buckets/manager'; +import { addLog } from '../../system/log'; +import { setCron } from '../../system/cron'; +import { checkTimerLock } from '../../system/timerLock/utils'; +import { TimerIdEnum } from '../../system/timerLock/constants'; + +export async function clearExpiredMinioFiles() { + try { + const now = new Date(); + + const expiredFiles = await MongoS3TTL.find({ expiredTime: { $lte: now } }).lean(); + if (expiredFiles.length === 0) { + addLog.info('No expired minio files to clean'); + return; + } + + addLog.info(`Found ${expiredFiles.length} expired minio files to clean`); + + const s3Manager = S3BucketManager.getInstance(); + let success = 0; + let fail = 0; + + for (const file of expiredFiles) { + try { + const bucket = (() => { + if (file.bucketName === process.env.S3_PUBLIC_BUCKET) { + return s3Manager.getPublicBucket(); + } + if (file.bucketName === process.env.S3_PRIVATE_BUCKET) { + return s3Manager.getPrivateBucket(); + } + throw new Error(`Unknown bucket name: ${file.bucketName}`); + })(); + + await bucket.delete(file.minioKey); + await MongoS3TTL.deleteOne({ _id: file._id }); + + success++; + addLog.info(`Deleted expired minio file: ${file.minioKey} from bucket: ${file.bucketName}`); + } catch (error) { + fail++; + addLog.error(`Failed to delete minio file: ${file.minioKey}`, error); + } + } + + addLog.info(`Minio TTL cleanup completed. Success: ${success}, Failed: ${fail}`); + } catch (error) { + addLog.error('Error in clearExpiredMinioFiles', error); + } +} + +export function clearExpiredMinioFilesCron() { + // 每小时执行一次 + setCron('0 */1 * * *', async () => { + if ( + await checkTimerLock({ + timerId: TimerIdEnum.clearExpiredMinioFiles, + lockMinuted: 59 + }) + ) { + await clearExpiredMinioFiles(); + } + }); +} diff --git a/packages/service/common/file/s3Ttl/schema.ts b/packages/service/common/file/s3Ttl/schema.ts new file mode 100644 index 000000000000..2919775b2de6 --- /dev/null +++ b/packages/service/common/file/s3Ttl/schema.ts @@ -0,0 +1,24 @@ +import { Schema, getMongoModel } from '../../mongo'; +import { type S3TtlSchemaType } from '@fastgpt/global/common/file/minioTtl/type'; + +const collectionName = 's3_ttl_files'; + +const S3TtlSchema = new Schema({ + bucketName: { + type: String, + required: true + }, + minioKey: { + type: String, + required: true + }, + expiredTime: { + type: Date, + required: true + } +}); + +S3TtlSchema.index({ expiredTime: 1 }); +S3TtlSchema.index({ bucketName: 1, minioKey: 1 }); + +export const MongoS3TTL = getMongoModel(collectionName, S3TtlSchema); diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts new file mode 100644 index 000000000000..3a48233aa326 --- /dev/null +++ b/packages/service/common/s3/buckets/base.ts @@ -0,0 +1,110 @@ +import { Client, type RemoveOptions, type CopyConditions, type LifecycleConfig } from 'minio'; +import { + defaultS3Options, + type ExtensionType, + Mimes, + type CreatePostPresignedUrlOptions, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3BucketName, + type S3Options +} from '../type'; +import type { BucketBasicOperationsType } from '../interface'; +import { createObjectKey, createTempObjectKey } from '../helpers'; +import path from 'node:path'; +import { MongoS3TTL } from 'common/file/s3TTL/schema'; + +export class S3BaseBucket implements BucketBasicOperationsType { + public client: Client; + + /** + * + * @param _bucket the bucket you want to operate + * @param options the options for the s3 client + * @param afterInits the function to be called after instantiating the s3 service + */ + constructor( + private readonly _bucket: S3BucketName, + private readonly afterInits?: (() => Promise | void)[], + public options: Partial = defaultS3Options + ) { + options = { ...defaultS3Options, ...options }; + this.options = options as S3Options; + this.client = new Client(options as S3Options); + + const init = async () => { + if (!(await this.exist())) { + await this.client.makeBucket(this._bucket); + } + await Promise.all(this.afterInits?.map((afterInit) => afterInit()) ?? []); + }; + init(); + } + + get name(): string { + return this._bucket; + } + + async move(src: string, dst: string, options?: CopyConditions): Promise { + const bucket = this.name; + await this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options); + await this.delete(src); + } + + copy(src: string, dst: string, options?: CopyConditions): ReturnType { + return this.client.copyObject(this.name, src, dst, options); + } + + exist(): Promise { + return this.client.bucketExists(this.name); + } + + async delete(objectKey: string, options?: RemoveOptions): Promise { + await this.client.removeObject(this.name, objectKey, options); + } + + get(): Promise { + throw new Error('Method not implemented.'); + } + + getLifecycle(): Promise { + return this.client.getBucketLifecycle(this.name); + } + + async createPostPresignedUrl( + params: CreatePostPresignedUrlParams, + options: CreatePostPresignedUrlOptions = {} + ): Promise { + const { temporary, ttl = 7 * 24 } = options; + const ext = path.extname(params.filename).toLowerCase() as ExtensionType; + const contentType = Mimes[ext] ?? 'application/octet-stream'; + const maxFileSize = this.options.maxFileSize as number; + const key = temporary ? createTempObjectKey(params) : createObjectKey(params); + + const policy = this.client.newPostPolicy(); + policy.setKey(key); + policy.setBucket(this.name); + policy.setContentType(contentType); + policy.setContentLengthRange(1, maxFileSize); + policy.setExpires(new Date(Date.now() + 10 * 60 * 1000)); + policy.setUserMetaData({ + filename: encodeURIComponent(params.filename), + visibility: params.visibility + }); + + const { formData, postURL } = await this.client.presignedPostPolicy(policy); + + if (temporary) { + await MongoS3TTL.create({ + minioKey: key, + bucketName: this.name, + expiredTime: new Date(Date.now() + ttl * 3.6e6) + }); + } + + return { + url: postURL, + fields: formData + }; + } +} diff --git a/packages/service/common/s3/buckets/manager.ts b/packages/service/common/s3/buckets/manager.ts new file mode 100644 index 000000000000..ac957ab07c47 --- /dev/null +++ b/packages/service/common/s3/buckets/manager.ts @@ -0,0 +1,23 @@ +import { type S3Options } from '../type'; +import { S3PublicBucket } from './public'; +import { S3PrivateBucket } from './private'; + +export class S3BucketManager { + private static instance: S3BucketManager; + private publicBucket: S3PublicBucket | null = null; + private privateBucket: S3PrivateBucket | null = null; + + private constructor() {} + + static getInstance(): S3BucketManager { + return (this.instance ??= new S3BucketManager()); + } + + getPublicBucket(options?: Partial): S3PublicBucket { + return (this.publicBucket ??= new S3PublicBucket(options)); + } + + getPrivateBucket(options?: Partial): S3PrivateBucket { + return (this.privateBucket ??= new S3PrivateBucket(options)); + } +} diff --git a/packages/service/common/s3/buckets/private.ts b/packages/service/common/s3/buckets/private.ts new file mode 100644 index 000000000000..05c80aab88df --- /dev/null +++ b/packages/service/common/s3/buckets/private.ts @@ -0,0 +1,19 @@ +import { S3BaseBucket } from './base'; +import { + S3Buckets, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../type'; + +export class S3PrivateBucket extends S3BaseBucket { + constructor(options?: Partial) { + super(S3Buckets.private, undefined, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return super.createPostPresignedUrl({ ...params, visibility: 'private' }); + } +} diff --git a/packages/service/common/s3/buckets/public.ts b/packages/service/common/s3/buckets/public.ts new file mode 100644 index 000000000000..e0c493dac706 --- /dev/null +++ b/packages/service/common/s3/buckets/public.ts @@ -0,0 +1,64 @@ +import { S3BaseBucket } from './base'; +import { createBucketPolicy } from '../helpers'; +import { + S3Buckets, + type CreatePostPresignedUrlOptions, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../type'; +import type { IPublicBucketOperations } from '../interface'; +import { lifecycleOfTemporaryAvatars } from '../lifecycle'; + +export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperations { + constructor(options?: Partial) { + super( + S3Buckets.public, + [ + // set bucket policy + async () => { + const bucket = this.name; + const policy = createBucketPolicy(bucket); + try { + await this.client.setBucketPolicy(bucket, policy); + } catch (error) { + // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error, + // maybe we can ignore the error, or we have other plan to handle this. + } + }, + // set bucket lifecycle + async () => { + const bucket = this.name; + await this.client.setBucketLifecycle(bucket, lifecycleOfTemporaryAvatars); + } + ], + options + ); + } + + createPublicUrl(objectKey: string): string { + const protocol = this.options.useSSL ? 'https' : 'http'; + const hostname = this.options.endPoint; + const port = this.options.port; + const bucket = this.name; + + const url = new URL(`${protocol}://${hostname}:${port}/${bucket}/${objectKey}`); + + if (this.options.externalBaseURL) { + const externalBaseURL = new URL(this.options.externalBaseURL); + + url.port = externalBaseURL.port; + url.hostname = externalBaseURL.hostname; + url.protocol = externalBaseURL.protocol; + } + + return url.toString(); + } + + override createPostPresignedUrl( + params: Omit, + options: CreatePostPresignedUrlOptions = {} + ): Promise { + return super.createPostPresignedUrl({ ...params, visibility: 'public' }, options); + } +} diff --git a/packages/service/common/s3/config.ts b/packages/service/common/s3/config.ts deleted file mode 100644 index 501d61be4b1a..000000000000 --- a/packages/service/common/s3/config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { S3ServiceConfig } from './type'; - -export const defualtS3Config: Omit = { - endPoint: process.env.S3_ENDPOINT || 'localhost', - port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, - useSSL: process.env.S3_USE_SSL === 'true', - accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', - secretKey: process.env.S3_SECRET_KEY || 'minioadmin', - externalBaseURL: process.env.S3_EXTERNAL_BASE_URL -}; diff --git a/packages/service/common/s3/const.ts b/packages/service/common/s3/const.ts deleted file mode 100644 index c91043dc1f96..000000000000 --- a/packages/service/common/s3/const.ts +++ /dev/null @@ -1,20 +0,0 @@ -export const mimeMap: Record = { - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.png': 'image/png', - '.gif': 'image/gif', - '.webp': 'image/webp', - '.svg': 'image/svg+xml', - '.pdf': 'application/pdf', - '.txt': 'text/plain', - '.json': 'application/json', - '.csv': 'text/csv', - '.zip': 'application/zip', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - '.doc': 'application/msword', - '.xls': 'application/vnd.ms-excel', - '.ppt': 'application/vnd.ms-powerpoint', - '.js': 'application/javascript' -}; diff --git a/packages/service/common/s3/controller.ts b/packages/service/common/s3/controller.ts deleted file mode 100644 index faaae7cdc91e..000000000000 --- a/packages/service/common/s3/controller.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { Client } from 'minio'; -import { - type FileMetadataType, - type PresignedUrlInput as UploadPresignedURLProps, - type UploadPresignedURLResponse, - type S3ServiceConfig -} from './type'; -import { defualtS3Config } from './config'; -import { randomBytes } from 'crypto'; -import { HttpProxyAgent } from 'http-proxy-agent'; -import { HttpsProxyAgent } from 'https-proxy-agent'; -import { extname } from 'path'; -import { addLog } from '../../common/system/log'; -import { getErrText } from '@fastgpt/global/common/error/utils'; -import { mimeMap } from './const'; - -export class S3Service { - private client: Client; - private config: S3ServiceConfig; - private initialized: boolean = false; - initFunction?: () => Promise; - - constructor(config?: Partial) { - this.config = { ...defualtS3Config, ...config } as S3ServiceConfig; - - this.client = new Client({ - endPoint: this.config.endPoint, - port: this.config.port, - useSSL: this.config.useSSL, - accessKey: this.config.accessKey, - secretKey: this.config.secretKey, - transportAgent: process.env.HTTP_PROXY - ? new HttpProxyAgent(process.env.HTTP_PROXY) - : process.env.HTTPS_PROXY - ? new HttpsProxyAgent(process.env.HTTPS_PROXY) - : undefined - }); - - this.initFunction = config?.initFunction; - } - - public async init() { - if (!this.initialized) { - if (!(await this.client.bucketExists(this.config.bucket))) { - addLog.debug(`Creating bucket: ${this.config.bucket}`); - await this.client.makeBucket(this.config.bucket); - } - - await this.initFunction?.(); - this.initialized = true; - } - } - - private generateFileId(): string { - return randomBytes(16).toString('hex'); - } - - private generateAccessUrl(filename: string): string { - const protocol = this.config.useSSL ? 'https' : 'http'; - const port = - this.config.port && this.config.port !== (this.config.useSSL ? 443 : 80) - ? `:${this.config.port}` - : ''; - - const externalBaseURL = this.config.externalBaseURL; - return externalBaseURL - ? `${externalBaseURL}/${this.config.bucket}/${encodeURIComponent(filename)}` - : `${protocol}://${this.config.endPoint}${port}/${this.config.bucket}/${encodeURIComponent(filename)}`; - } - - uploadFile = async (fileBuffer: Buffer, originalFilename: string): Promise => { - await this.init(); - const inferContentType = (filename: string) => { - const ext = extname(filename).toLowerCase(); - return mimeMap[ext] || 'application/octet-stream'; - }; - - if (this.config.maxFileSize && fileBuffer.length > this.config.maxFileSize) { - return Promise.reject( - `File size ${fileBuffer.length} exceeds limit ${this.config.maxFileSize}` - ); - } - - const fileId = this.generateFileId(); - const objectName = `${fileId}-${originalFilename}`; - const uploadTime = new Date(); - - const contentType = inferContentType(originalFilename); - await this.client.putObject(this.config.bucket, objectName, fileBuffer, fileBuffer.length, { - 'Content-Type': contentType, - 'Content-Disposition': `attachment; filename="${encodeURIComponent(originalFilename)}"`, - 'x-amz-meta-original-filename': encodeURIComponent(originalFilename), - 'x-amz-meta-upload-time': uploadTime.toISOString() - }); - - const metadata: FileMetadataType = { - fileId, - originalFilename, - contentType, - size: fileBuffer.length, - uploadTime, - accessUrl: this.generateAccessUrl(objectName) - }; - - return metadata; - }; - - generateUploadPresignedURL = async ({ - filepath, - contentType, - metadata, - filename - }: UploadPresignedURLProps): Promise => { - await this.init(); - const objectName = `${filepath}/${filename}`; - - try { - const policy = this.client.newPostPolicy(); - - policy.setBucket(this.config.bucket); - policy.setKey(objectName); - if (contentType) { - policy.setContentType(contentType); - } - if (this.config.maxFileSize) { - policy.setContentLengthRange(1, this.config.maxFileSize); - } - policy.setExpires(new Date(Date.now() + 10 * 60 * 1000)); // 10 mins - - policy.setUserMetaData({ - 'original-filename': encodeURIComponent(filename), - 'upload-time': new Date().toISOString(), - ...metadata - }); - - const { postURL, formData } = await this.client.presignedPostPolicy(policy); - - const response: UploadPresignedURLResponse = { - objectName, - uploadUrl: postURL, - formData - }; - - return response; - } catch (error) { - addLog.error('Failed to generate Upload Presigned URL', error); - return Promise.reject(`Failed to generate Upload Presigned URL: ${getErrText(error)}`); - } - }; - - generateDownloadUrl = (objectName: string): string => { - const pathParts = objectName.split('/'); - const encodedParts = pathParts.map((part) => encodeURIComponent(part)); - const encodedObjectName = encodedParts.join('/'); - return `${this.config.bucket}/${encodedObjectName}`; - }; - - getFile = async (objectName: string): Promise => { - const stat = await this.client.statObject(this.config.bucket, objectName); - - if (stat.size > 0) { - const accessUrl = this.generateDownloadUrl(objectName); - return accessUrl; - } - - return Promise.reject(`File ${objectName} not found`); - }; -} diff --git a/packages/service/common/s3/helpers.ts b/packages/service/common/s3/helpers.ts new file mode 100644 index 000000000000..71814d83bc4b --- /dev/null +++ b/packages/service/common/s3/helpers.ts @@ -0,0 +1,58 @@ +import crypto from 'node:crypto'; +import { type CreateObjectKeyParams } from './type'; +import dayjs from 'dayjs'; + +/** + * use public policy or just a custom policy + * + * @default policy public policy + * @param bucket bucket name + * @returns the policy string + */ +export const createBucketPolicy = ( + bucket: string, + policy?: 'public' | Record +): string => { + if (typeof policy === 'string' && policy !== 'public') { + throw new Error("'policy' only can be assigned to 'public' if typeof 'policy' is string"); + } + + switch (typeof policy) { + case 'string': + case 'undefined': + default: + return JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: '*', + Action: 's3:GetObject', + Resource: `arn:aws:s3:::${bucket}/*` + } + ] + }); + case 'object': + if (policy === null) { + throw new Error("Bucket policy can't be null"); + } + return JSON.stringify(policy); + } +}; + +/** + * create s3 object key by source, team ID and filename + */ +export function createObjectKey({ source, teamId, filename }: CreateObjectKeyParams): string { + const date = dayjs().format('YYYY_MM_DD'); + const id = crypto.randomBytes(16).toString('hex'); + return `${source}/${teamId}/${date}/${id}_${filename}`; +} + +/** + * create temporary s3 object key by source, team ID and filename + */ +export function createTempObjectKey(params: CreateObjectKeyParams): string { + const origin = createObjectKey(params); + return `temp/${origin}`; +} diff --git a/packages/service/common/s3/index.ts b/packages/service/common/s3/index.ts deleted file mode 100644 index 761bd564097e..000000000000 --- a/packages/service/common/s3/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { S3Service } from './controller'; - -export const PluginS3Service = (() => { - if (!global.pluginS3Service) { - global.pluginS3Service = new S3Service({ - bucket: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', - maxFileSize: 50 * 1024 * 1024 // 50MB - }); - } - - return global.pluginS3Service; -})(); - -declare global { - var pluginS3Service: S3Service; -} diff --git a/packages/service/common/s3/interface.ts b/packages/service/common/s3/interface.ts new file mode 100644 index 000000000000..b9910b4179c4 --- /dev/null +++ b/packages/service/common/s3/interface.ts @@ -0,0 +1,20 @@ +import { type LifecycleConfig, type Client, type CopyConditions, type RemoveOptions } from 'minio'; +import type { CreateObjectKeyParams, CreatePostPresignedUrlOptions } from './type'; + +export interface BucketBasicOperationsType { + get name(): string; + get(): Promise; + exist(): Promise; + delete(objectKey: string, options?: RemoveOptions): Promise; + move(src: string, dst: string, options?: CopyConditions): Promise; + copy(src: string, dst: string, options?: CopyConditions): ReturnType; + getLifecycle(): Promise; + createPostPresignedUrl( + params: CreateObjectKeyParams, + options?: CreatePostPresignedUrlOptions + ): Promise<{ url: string; fields: Record }>; +} + +export interface IPublicBucketOperations { + createPublicUrl(objectKey: string): string; +} diff --git a/packages/service/common/s3/lifecycle.ts b/packages/service/common/s3/lifecycle.ts new file mode 100644 index 000000000000..12c28c1ca746 --- /dev/null +++ b/packages/service/common/s3/lifecycle.ts @@ -0,0 +1,23 @@ +import { type LifecycleRule, type LifecycleConfig } from 'minio'; + +export function createLifeCycleConfig(rule: LifecycleRule): LifecycleConfig { + return { + Rule: [rule] + }; +} + +export function assembleLifeCycleConfigs(...configs: LifecycleConfig[]): LifecycleConfig { + if (configs.length === 0) return { Rule: [] }; + return { + Rule: configs.flatMap((config) => config.Rule) + }; +} + +export const lifecycleOfTemporaryAvatars = createLifeCycleConfig({ + ID: 'Temporary Avatars Rule', + Prefix: 'temp/avatar/', + Status: 'Enabled', + Expiration: { + Days: 1 + } +}); diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts new file mode 100644 index 000000000000..1c57baf4da6b --- /dev/null +++ b/packages/service/common/s3/sources/avatar.ts @@ -0,0 +1,74 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + S3APIPrefix, + type CreatePostPresignedUrlOptions, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../type'; +import type { S3PublicBucket } from '../buckets/public'; +import { MongoS3TTL } from '../../file/s3TTL/schema'; + +class S3AvatarSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.avatar, true, options); + } + + static getInstance(options?: Partial): S3AvatarSource { + return S3BaseSource._getInstance(S3AvatarSource, options); + } + + override createPostPresignedUrl( + params: Omit, + options: CreatePostPresignedUrlOptions = {} + ): Promise { + return this.bucket.createPostPresignedUrl( + { + ...params, + source: S3Sources.avatar + }, + options + ); + } + + createPublicUrl(objectKey: string): string { + return this.bucket.createPublicUrl(objectKey); + } + + createAvatarObjectKey(avatarWithPrefix: string): string { + return avatarWithPrefix.replace(S3APIPrefix.avatar, ''); + } + + async removeAvatar(avatarWithPrefix: string): Promise { + const avatarObjectKey = this.createAvatarObjectKey(avatarWithPrefix); + await MongoS3TTL.deleteOne({ minioKey: avatarObjectKey, bucketName: this.bucketName }); + await this.bucket.delete(avatarObjectKey); + } + + async moveAvatarFromTemp(tempAvatarWithPrefix: string): Promise { + const tempAvatarObjectKey = this.createAvatarObjectKey(tempAvatarWithPrefix); + const avatarObjectKey = tempAvatarObjectKey.replace(`${S3Sources.temp}/`, ''); + + try { + const file = await MongoS3TTL.findOne({ + bucketName: this.bucketName, + minioKey: tempAvatarObjectKey + }); + if (file) { + file.set({ expiredTime: undefined, minioKey: avatarObjectKey }); + await file.save(); + } + } catch (error) { + console.error('Failed to convert TTL to permanent:', error); + } + + await this.bucket.move(tempAvatarObjectKey, avatarObjectKey); + + return S3APIPrefix.avatar + avatarObjectKey; + } +} + +export function getS3AvatarSource() { + return S3AvatarSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/base.ts b/packages/service/common/s3/sources/base.ts new file mode 100644 index 000000000000..98e3e8dcfce4 --- /dev/null +++ b/packages/service/common/s3/sources/base.ts @@ -0,0 +1,58 @@ +import { S3BucketManager } from '../buckets/manager'; +import type { S3PrivateBucket } from '../buckets/private'; +import type { S3PublicBucket } from '../buckets/public'; +import type { + CreatePostPresignedUrlParams, + CreatePostPresignedUrlResult, + S3Options, + S3SourceType +} from '../type'; + +type Bucket = S3PublicBucket | S3PrivateBucket; + +export abstract class S3BaseSource { + protected bucket: T; + protected static instances: Map = new Map(); + + constructor( + protected readonly source: S3SourceType, + protected readonly pub: boolean = false, + options?: Partial + ) { + const manager = S3BucketManager.getInstance(); + switch (pub) { + case true: + this.bucket = manager.getPublicBucket(options) as T; + break; + case false: + this.bucket = manager.getPrivateBucket(options) as T; + } + } + + abstract createPostPresignedUrl( + params: Omit + ): Promise; + + protected static _getInstance( + constructor: new (options?: Partial) => T, + options?: Partial + ): T { + const className = constructor.name; + if (!S3BaseSource.instances.has(className)) { + S3BaseSource.instances.set(className, new constructor(options)); + } + return S3BaseSource.instances.get(className) as T; + } + + protected getBucket(): T { + return this.bucket; + } + + get bucketName(): string { + return this.bucket.name; + } + + isBucketExist(): Promise { + return this.bucket.exist(); + } +} diff --git a/packages/service/common/s3/sources/chat.ts b/packages/service/common/s3/sources/chat.ts new file mode 100644 index 000000000000..e4ef8914f124 --- /dev/null +++ b/packages/service/common/s3/sources/chat.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../type'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3ChatSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.chat, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.chat }); + } + + static getInstance(options?: Partial): S3ChatSource { + return S3BaseSource._getInstance(S3ChatSource, options); + } +} + +export function getS3ChatSource() { + return S3ChatSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/dataset-image.ts b/packages/service/common/s3/sources/dataset-image.ts new file mode 100644 index 000000000000..5bf71d58b8c1 --- /dev/null +++ b/packages/service/common/s3/sources/dataset-image.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../type'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3DatasetImageSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources['dataset-image'], false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources['dataset-image'] }); + } + + static getInstance(options?: Partial): S3DatasetImageSource { + return S3BaseSource._getInstance(S3DatasetImageSource, options); + } +} + +export function getS3DatasetImageSource() { + return S3DatasetImageSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/dataset.ts b/packages/service/common/s3/sources/dataset.ts new file mode 100644 index 000000000000..06a9cd3881f2 --- /dev/null +++ b/packages/service/common/s3/sources/dataset.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../type'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3DatasetSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.dataset, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.dataset }); + } + + static getInstance(options?: Partial): S3DatasetSource { + return S3BaseSource._getInstance(S3DatasetSource, options); + } +} + +export function getS3DatasetSource() { + return S3DatasetSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/index.ts b/packages/service/common/s3/sources/index.ts new file mode 100644 index 000000000000..a19622639276 --- /dev/null +++ b/packages/service/common/s3/sources/index.ts @@ -0,0 +1,24 @@ +import { getS3AvatarSource } from './avatar'; +import { getS3ChatSource } from './chat'; +import { getS3DatasetSource } from './dataset'; +import { getS3DatasetImageSource } from './dataset-image'; +import { getS3InvoiceSource } from './invoice'; +import { getS3RawtextSource } from './rawtext'; + +export function registerSources() { + getS3AvatarSource(); + getS3ChatSource(); + getS3DatasetImageSource(); + getS3DatasetSource(); + getS3InvoiceSource(); + getS3RawtextSource(); +} + +export { + getS3AvatarSource, + getS3ChatSource, + getS3DatasetImageSource, + getS3DatasetSource, + getS3InvoiceSource, + getS3RawtextSource +}; diff --git a/packages/service/common/s3/sources/invoice.ts b/packages/service/common/s3/sources/invoice.ts new file mode 100644 index 000000000000..57cc7bfff86c --- /dev/null +++ b/packages/service/common/s3/sources/invoice.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../type'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3InvoiceSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.invoice, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.invoice }); + } + + static getInstance(options?: Partial): S3InvoiceSource { + return S3BaseSource._getInstance(S3InvoiceSource, options); + } +} + +export function getS3InvoiceSource() { + return S3InvoiceSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/rawtext.ts b/packages/service/common/s3/sources/rawtext.ts new file mode 100644 index 000000000000..8b505a32475c --- /dev/null +++ b/packages/service/common/s3/sources/rawtext.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../type'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3RawtextSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.rawtext, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.rawtext }); + } + + static getInstance(options?: Partial): S3RawtextSource { + return S3BaseSource._getInstance(S3RawtextSource, options); + } +} + +export function getS3RawtextSource() { + return S3RawtextSource.getInstance(); +} diff --git a/packages/service/common/s3/type.ts b/packages/service/common/s3/type.ts index a480530493e8..09e44d52c37f 100644 --- a/packages/service/common/s3/type.ts +++ b/packages/service/common/s3/type.ts @@ -1,49 +1,106 @@ import type { ClientOptions } from 'minio'; +import { HttpProxyAgent } from 'http-proxy-agent'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import { z } from 'zod'; -export type S3ServiceConfig = { - bucket: string; - externalBaseURL?: string; - /** - * Unit: Byte - */ - maxFileSize?: number; - /** - * for executing some init function for the s3 service - */ - initFunction?: () => Promise; -} & ClientOptions; - -export type FileMetadataType = { - fileId: string; - originalFilename: string; - contentType: string; - size: number; - uploadTime: Date; - accessUrl: string; -}; - -export type PresignedUrlInput = { - filepath: string; - filename: string; - contentType?: string; - metadata?: Record; -}; - -export type UploadPresignedURLResponse = { - objectName: string; - uploadUrl: string; - formData: Record; -}; - -export type FileUploadInput = { - buffer: Buffer; - filename: string; -}; - -export enum PluginTypeEnum { - tool = 'tool' -} - -export const PluginFilePath = { - [PluginTypeEnum.tool]: 'plugin/tools' -}; +export const S3MetadataSchema = z.object({ + filename: z.string(), + uploadedAt: z.date(), + accessUrl: z.string(), + contentType: z.string(), + id: z.string().length(32), + size: z.number().positive() +}); +export type S3Metadata = z.infer; + +export const Mimes = { + '.gif': 'image/gif', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + + '.csv': 'text/csv', + '.txt': 'text/plain', + + '.pdf': 'application/pdf', + '.zip': 'application/zip', + '.json': 'application/json', + '.doc': 'application/msword', + '.js': 'application/javascript', + '.xls': 'application/vnd.ms-excel', + '.ppt': 'application/vnd.ms-powerpoint', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation' +} as const; +export type ContentType = (typeof Mimes)[keyof typeof Mimes]; +export type ExtensionType = keyof typeof Mimes; + +export const defaultS3Options: { externalBaseURL?: string; maxFileSize?: number } & ClientOptions = + { + maxFileSize: 1024 ** 3, // 1GB + + useSSL: process.env.S3_USE_SSL === 'true', + endPoint: process.env.S3_ENDPOINT || 'localhost', + externalBaseURL: process.env.S3_EXTERNAL_BASE_URL, + accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', + secretKey: process.env.S3_SECRET_KEY || 'minioadmin', + port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, + transportAgent: process.env.HTTP_PROXY + ? new HttpProxyAgent(process.env.HTTP_PROXY) + : process.env.HTTPS_PROXY + ? new HttpsProxyAgent(process.env.HTTPS_PROXY) + : undefined + }; +export type S3Options = typeof defaultS3Options; + +export const S3Buckets = { + plugin: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', + public: process.env.S3_PUBLIC_BUCKET || 'fastgpt-public', + private: process.env.S3_PRIVATE_BUCKET || 'fastgpt-private' +} as const; +export type S3BucketName = (typeof S3Buckets)[keyof typeof S3Buckets]; + +export const S3SourcesSchema = z.enum([ + 'avatar', + 'chat', + 'dataset', + 'dataset-image', + 'invoice', + 'rawtext', + 'temp' +]); +export const S3Sources = S3SourcesSchema.enum; +export type S3SourceType = z.infer; + +export const CreateObjectKeyParamsSchema = z.object({ + filename: z.string().min(1), + source: S3SourcesSchema, + teamId: z.string().length(16) +}); +export type CreateObjectKeyParams = z.infer; + +export const CreatePostPresignedUrlParamsSchema = z.object({ + ...CreateObjectKeyParamsSchema.shape, + visibility: z.enum(['public', 'private']).default('private') +}); +export type CreatePostPresignedUrlParams = z.infer; + +export const CreatePostPresignedUrlOptionsSchema = z.object({ + temporary: z.boolean().optional(), + ttl: z.number().positive().optional() // TTL in Hours, default 7 * 24 +}); +export type CreatePostPresignedUrlOptions = z.infer; + +export const CreatePostPresignedUrlResultSchema = z.object({ + url: z.string().min(1), + fields: z.record(z.string(), z.string()) +}); +export type CreatePostPresignedUrlResult = z.infer; + +export const S3APIPrefix = { + avatar: '/api/system/img/' +} as const; +export type S3APIPrefixType = (typeof S3APIPrefix)[keyof typeof S3APIPrefix]; diff --git a/packages/service/common/system/timerLock/constants.ts b/packages/service/common/system/timerLock/constants.ts index 76189686c775..2768d0085a19 100644 --- a/packages/service/common/system/timerLock/constants.ts +++ b/packages/service/common/system/timerLock/constants.ts @@ -8,7 +8,8 @@ export enum TimerIdEnum { notification = 'notification', clearExpiredRawTextBuffer = 'clearExpiredRawTextBuffer', - clearExpiredDatasetImage = 'clearExpiredDatasetImage' + clearExpiredDatasetImage = 'clearExpiredDatasetImage', + clearExpiredMinioFiles = 'clearExpiredMinioFiles' } export enum LockNotificationEnum { diff --git a/packages/service/package.json b/packages/service/package.json index eb367ff8ed38..41438d10f7af 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -54,7 +54,8 @@ "tiktoken": "1.0.17", "tunnel": "^0.0.6", "turndown": "^7.1.2", - "winston": "^3.17.0" + "winston": "^3.17.0", + "zod": "^3.24.2" }, "devDependencies": { "@types/cookie": "^0.5.2", diff --git a/packages/web/common/file/hooks/useUploadAvatar.tsx b/packages/web/common/file/hooks/useUploadAvatar.tsx new file mode 100644 index 000000000000..10c8525e79e1 --- /dev/null +++ b/packages/web/common/file/hooks/useUploadAvatar.tsx @@ -0,0 +1,96 @@ +import { base64ToFile, fileToBase64 } from '../utils'; +import { compressBase64Img } from '../img'; +import { useToast } from '../../../hooks/useToast'; +import { useCallback, useRef, useTransition } from 'react'; +import { useTranslation } from 'next-i18next'; +import { type CreatePostPresignedUrlResult } from '../../../../service/common/s3/type'; + +export const useUploadAvatar = ( + api: (params: { filename: string; temporay: boolean }) => Promise, + { + temporay = false, + onSuccess + }: { + temporay?: boolean; + onSuccess?: (avatar: string) => void; + } = {} +) => { + const { toast } = useToast(); + const { t } = useTranslation(); + const [uploading, startUpload] = useTransition(); + const uploadAvatarRef = useRef(null); + + const handleFileSelectorOpen = useCallback(() => { + if (!uploadAvatarRef.current) return; + uploadAvatarRef.current.click(); + }, []); + + // manually upload avatar + const handleUploadAvatar = useCallback( + async (file: File) => { + if (!file.name.match(/\.(jpg|png|jpeg)$/)) { + toast({ title: t('account_info:avatar_can_only_select_jpg_png'), status: 'warning' }); + return; + } + + startUpload(async () => { + const compressed = base64ToFile( + await compressBase64Img({ + base64Img: await fileToBase64(file), + maxW: 300, + maxH: 300 + }), + file.name + ); + const { url, fields } = await api({ filename: file.name, temporay }); + const formData = new FormData(); + Object.entries(fields).forEach(([k, v]) => formData.set(k, v)); + formData.set('file', compressed); + await fetch(url, { method: 'POST', body: formData }); // 204 + + const avatar = `/api/system/img/${fields.key}`; + onSuccess?.(avatar); + }); + }, + [t, toast, api, temporay, onSuccess] + ); + + const onUploadAvatarChange = useCallback( + async (e: React.ChangeEvent) => { + const files = e.target.files; + + if (!files || files.length === 0) { + e.target.value = ''; + return; + } + if (files.length > 1) { + toast({ title: t('account_info:avatar_can_only_select_one'), status: 'warning' }); + e.target.value = ''; + return; + } + const file = files[0]!; + handleUploadAvatar(file); + }, + [t, toast, handleUploadAvatar] + ); + + const Component = useCallback(() => { + return ( + + ); + }, [onUploadAvatarChange]); + + return { + uploading, + Component, + handleFileSelectorOpen, + handleUploadAvatar + }; +}; diff --git a/packages/web/common/file/utils.ts b/packages/web/common/file/utils.ts index 7fd10744530f..f4107301d585 100644 --- a/packages/web/common/file/utils.ts +++ b/packages/web/common/file/utils.ts @@ -99,3 +99,24 @@ async function detectFileEncoding(file: File): Promise { return encoding || 'utf-8'; } + +export const fileToBase64 = (file: File) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => resolve(reader.result as string); + reader.onerror = (error) => reject(error); + }); +}; + +export const base64ToFile = (base64: string, filename: string) => { + const arr = base64.split(','); + const mime = arr[0].match(/:(.*?);/)?.[1]; + const bstr = atob(arr[1]); + let n = bstr.length; + const u8arr = new Uint8Array(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n); + } + return new File([u8arr], filename, { type: mime }); +}; diff --git a/packages/web/i18n/en/account_info.json b/packages/web/i18n/en/account_info.json index 15a520732bef..c420e057d285 100644 --- a/packages/web/i18n/en/account_info.json +++ b/packages/web/i18n/en/account_info.json @@ -9,6 +9,8 @@ "app_amount": "App amount", "avatar": "Avatar", "avatar_selection_exception": "Abnormal avatar selection", + "avatar_can_only_select_one": "Avatar can only select one picture", + "avatar_can_only_select_jpg_png": "Avatar can only select jpg or png format", "balance": "balance", "billing_standard": "Standards", "cancel": "Cancel", diff --git a/packages/web/i18n/zh-CN/account_info.json b/packages/web/i18n/zh-CN/account_info.json index cd20076cdcc6..7822d8765995 100644 --- a/packages/web/i18n/zh-CN/account_info.json +++ b/packages/web/i18n/zh-CN/account_info.json @@ -9,6 +9,8 @@ "app_amount": "应用数量", "avatar": "头像", "avatar_selection_exception": "头像选择异常", + "avatar_can_only_select_one": "头像只能选择一张图片", + "avatar_can_only_select_jpg_png": "头像只能选择 jpg 或 png 格式", "balance": "余额", "billing_standard": "计费标准", "cancel": "取消", diff --git a/packages/web/i18n/zh-Hant/account_info.json b/packages/web/i18n/zh-Hant/account_info.json index 9c5560d48d91..5646df649802 100644 --- a/packages/web/i18n/zh-Hant/account_info.json +++ b/packages/web/i18n/zh-Hant/account_info.json @@ -9,6 +9,8 @@ "app_amount": "應用數量", "avatar": "頭像", "avatar_selection_exception": "頭像選擇異常", + "avatar_can_only_select_one": "頭像只能選擇一張圖片", + "avatar_can_only_select_jpg_png": "頭像只能選擇 jpg 或 png 格式", "balance": "餘額", "billing_standard": "計費標準", "cancel": "取消", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7da780021d45..066581a4aea0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -282,6 +282,9 @@ importers: winston: specifier: ^3.17.0 version: 3.17.0 + zod: + specifier: ^3.24.2 + version: 3.25.51 devDependencies: '@types/cookie': specifier: ^0.5.2 @@ -12231,8 +12234,8 @@ snapshots: express-rate-limit: 7.5.0(express@5.1.0) pkce-challenge: 5.0.0 raw-body: 3.0.0 - zod: 3.24.2 - zod-to-json-schema: 3.24.5(zod@3.24.2) + zod: 3.25.51 + zod-to-json-schema: 3.24.5(zod@3.25.51) transitivePeerDependencies: - supports-color @@ -21900,9 +21903,9 @@ snapshots: - terser - typescript - zod-to-json-schema@3.24.5(zod@3.24.2): + zod-to-json-schema@3.24.5(zod@3.25.51): dependencies: - zod: 3.24.2 + zod: 3.25.51 zod-validation-error@3.4.0(zod@3.25.51): dependencies: diff --git a/projects/app/.env.template b/projects/app/.env.template index fd51a63b3852..eb383699b3cf 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -41,7 +41,10 @@ S3_PORT=9000 S3_USE_SSL=false S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin -S3_PLUGIN_BUCKET=fastgpt-plugin # 插件文件存储bucket +S3_PLUGIN_BUCKET=fastgpt-plugins # 插件文件存储bucket +S3_PUBLIC_BUCKET=fastgpt-public # 插件文件存储公开桶 +S3_PRIVATE_BUCKET=fastgpt-private # 插件文件存储公开桶 + # Redis URL REDIS_URL=redis://default:mypassword@127.0.0.1:6379 # mongo 数据库连接参数，本地开发连接远程数据库时，可能需要增加 directConnection=true 参数，才能连接上。 diff --git a/projects/app/src/components/common/Modal/EditResourceModal.tsx b/projects/app/src/components/common/Modal/EditResourceModal.tsx index 85c8d06d2c49..e317311f795f 100644 --- a/projects/app/src/components/common/Modal/EditResourceModal.tsx +++ b/projects/app/src/components/common/Modal/EditResourceModal.tsx @@ -1,13 +1,14 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { ModalFooter, ModalBody, Input, Button, Box, Textarea, HStack } from '@chakra-ui/react'; import MyModal from '@fastgpt/web/components/common/MyModal/index'; import { useTranslation } from 'next-i18next'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; import { useForm } from 'react-hook-form'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import Avatar from '@fastgpt/web/components/common/Avatar'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; export type EditResourceInfoFormType = { id: string; @@ -41,14 +42,18 @@ const EditResourceModal = ({ } ); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false - }); + const afterUploadAvatar = useCallback( + (avatar: string) => { + console.log('avatar', avatar); + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen: handleAvatarSelectorOpen } = + useUploadAvatar(getUploadAvatarPresignedUrl, { + temporay: true, + onSuccess: afterUploadAvatar + }); return ( @@ -64,7 +69,7 @@ const EditResourceModal = ({ h={'2rem'} cursor={'pointer'} borderRadius={'sm'} - onClick={onOpenSelectFile} + onClick={handleAvatarSelectorOpen} /> - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> + ); }; diff --git a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx index db6f4574b403..0e0a0e95a114 100644 --- a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx +++ b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx @@ -1,7 +1,6 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { useForm } from 'react-hook-form'; import { useTranslation } from 'next-i18next'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { @@ -21,6 +20,8 @@ import { type CreateTeamProps } from '@fastgpt/global/support/user/team/controll import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants'; import Icon from '@fastgpt/web/components/common/Icon'; import dynamic from 'next/dynamic'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; const UpdateContact = dynamic(() => import('@/components/support/user/inform/UpdateContactModal')); export type EditTeamFormDataType = CreateTeamProps & { @@ -50,15 +51,6 @@ function EditModal({ const avatar = watch('avatar'); const notificationAccount = watch('notificationAccount'); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png,.svg', - multiple: false - }); - const { mutate: onclickCreate, isLoading: creating } = useRequest({ mutationFn: async (data: CreateTeamProps) => { return postCreateTeam(data); @@ -88,6 +80,19 @@ function EditModal({ const { isOpen: isOpenContact, onClose: onCloseContact, onOpen: onOpenContact } = useDisclosure(); + const afterUploadAvatar = useCallback( + (avatar: string) => { + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); + return ( {t('account_team:set_name_avatar')} + )} - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> {isOpenContact && ( onFileSelect(avatar), [onFileSelect]); const { - File: SelectFileComponent, - onOpen: onOpenSelectFile, - onSelectImage, - loading - } = useSelectFile({ - fileType: 'image/*', - multiple: false, - maxCount: 1 - }); - - // validate file size - const validateFile = useMemoizedFn((file: File): string | null => { - if (file.size > maxSizeBytes) { - return t('chat:setting.copyright.file_size_exceeds_limit', { - maxSize: formatFileSize(maxSizeBytes) - }); - } - return null; + Component: SelectFileComponent, + uploading: loading, + handleFileSelectorOpen: onOpenSelectFile, + handleUploadAvatar: handleFileSelect + } = useUploadAvatar(getUploadAvatarPresignedUrl, { + temporay: true, + onSuccess: afterUploadAvatar }); // handle file select - immediate upload if enabled - const handleFileSelect = useMemoizedFn(async (files: File[]) => { - const file = files[0]; - - const validationError = validateFile(file); - if (validationError) { - toast({ - status: 'warning', - title: validationError - }); - } - - try { - // 立即上传文件，带TTL - const url = await onSelectImage([file], { maxW: 1000, maxH: 1000 }); - onFileSelect(url); - } catch (error) { - console.error('Failed to upload file:', error); - } - }); + // const handleFileSelect = useMemoizedFn(async (files: File[]) => { + // const file = files[0]; + + // const validationError = validateFile(file); + // if (validationError) { + // toast({ + // status: 'warning', + // title: validationError + // }); + // } + + // try { + // // 立即上传文件，带TTL + // const url = await onSelectImage([file], { maxW: 1000, maxH: 1000 }); + // onFileSelect(url); + // } catch (error) { + // console.error('Failed to upload file:', error); + // } + // }); // 拖拽处理 const handleDragEnter = useMemoizedFn((e: React.DragEvent) => { @@ -106,7 +109,7 @@ export const useImageUpload = ({ maxSize, onFileSelect }: UseImageUploadProps) = if (e.dataTransfer.files && e.dataTransfer.files.length > 0) { const files = Array.from(e.dataTransfer.files); - await handleFileSelect(files); + await handleFileSelect(files[0]); } }); diff --git a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx index bf652689ac71..d89119b22304 100644 --- a/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx +++ b/projects/app/src/pageComponents/chat/ChatSetting/ImageUpload/index.tsx @@ -80,7 +80,7 @@ const ImageUpload = ({ return ( - + import('@/pageComponents/account/info/RedeemCouponModal'), { ssr: false @@ -140,14 +142,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { onClose: onCloseUpdateContact, onOpen: onOpenUpdateContact } = useDisclosure(); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false - }); const onclickSave = useCallback( async (data: UserType) => { @@ -164,6 +158,20 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { [reset, t, toast, updateUserInfo] ); + const afterUploadAvatar = useCallback( + (avatar: string) => { + if (!userInfo) return; + onclickSave({ ...userInfo, avatar }); + }, + [onclickSave, userInfo] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); + const labelStyles: BoxProps = { flex: '0 0 80px', color: 'var(--light-general-on-surface-lowest, var(--Gray-Modern-500, #667085))', @@ -241,6 +249,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} + {isPc ? ( {t('account_info:avatar')}  @@ -253,7 +262,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { border={theme.borders.base} overflow={'hidden'} boxShadow={'0 0 5px rgba(0,0,0,0.1)'} - onClick={onOpenSelectFile} + onClick={handleFileSelectorOpen} > @@ -264,7 +273,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { flexDirection={'column'} alignItems={'center'} cursor={'pointer'} - onClick={onOpenSelectFile} + onClick={handleFileSelectorOpen} > void }) => { )} + {feConfigs?.isPlus && ( {t('account_info:member_name')}  @@ -334,21 +344,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} {isOpenUpdatePsw && } {isOpenUpdateContact && } - - onSelectImage(e, { - maxW: 300, - maxH: 300, - callback: (src) => { - if (!userInfo) return; - onclickSave({ - ...userInfo, - avatar: src - }); - } - }) - } - /> ); }; diff --git a/projects/app/src/pages/api/common/file/updateAvatar.ts b/projects/app/src/pages/api/common/file/updateAvatar.ts new file mode 100644 index 000000000000..54d4b1cea491 --- /dev/null +++ b/projects/app/src/pages/api/common/file/updateAvatar.ts @@ -0,0 +1,28 @@ +import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import { NextAPI } from '@/service/middleware/entry'; +import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; + +export type updateAvatarQuery = {}; + +export type updateAvatarBody = { + filename: string; + temporay: boolean; +}; + +export type updateAvatarResponse = CreatePostPresignedUrlResult; + +async function handler( + req: ApiRequestProps, + _: ApiResponseType +): Promise { + const { filename, temporay } = req.body; + const { teamId } = await authCert({ req, authToken: true }); + return await getS3AvatarSource().createPostPresignedUrl( + { teamId, filename }, + { temporay, ttl: 1 } + ); +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/app/update.ts b/projects/app/src/pages/api/core/app/update.ts index 5709f8ccc7f9..0ab2cd802092 100644 --- a/projects/app/src/pages/api/core/app/update.ts +++ b/projects/app/src/pages/api/core/app/update.ts @@ -27,6 +27,7 @@ import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nAppType } from '@fastgpt/service/support/user/audit/util'; import { i18nT } from '@fastgpt/web/i18n/utils'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; export type AppUpdateQuery = { appId: string; @@ -111,13 +112,19 @@ async function handler(req: ApiRequestProps) { nodes }); - await refreshSourceAvatar(avatar, app.avatar, session); + const permanentAvatar = await (async () => { + if (avatar) { + const s3AvatarSource = getS3AvatarSource(); + await s3AvatarSource.removeAvatar(app.avatar); // 移除旧的头像 + return await s3AvatarSource.moveAvatarFromTemp(avatar); // 永久化临时的预览头像 + } + })(); if (app.type === AppTypeEnum.toolSet && avatar) { await MongoApp.updateMany( { parentId: appId, teamId: app.teamId }, { - avatar + avatar: permanentAvatar }, { session } ); @@ -129,7 +136,7 @@ async function handler(req: ApiRequestProps) { ...parseParentIdInMongo(parentId), ...(name && { name }), ...(type && { type }), - ...(avatar && { avatar }), + ...(avatar && { avatar: permanentAvatar }), ...(intro !== undefined && { intro }), ...(teamTags && { teamTags }), ...(nodes && { diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts index a2b0611ad353..5cd9ad7d51e7 100644 --- a/projects/app/src/pages/api/core/dataset/update.ts +++ b/projects/app/src/pages/api/core/dataset/update.ts @@ -39,6 +39,7 @@ import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model'; import { computedCollectionChunkSettings } from '@fastgpt/global/core/dataset/training/utils'; import { getResourceOwnedClbs } from '@fastgpt/service/support/permission/controller'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; export type DatasetUpdateQuery = {}; export type DatasetUpdateResponse = any; @@ -202,12 +203,20 @@ async function handler( return flattenObjectWithConditions(apiDatasetServer); })(); + const permanentAvatar = await (async () => { + if (avatar) { + const s3AvatarSource = getS3AvatarSource(); + await s3AvatarSource.removeAvatar(dataset.avatar); // 移除旧的头像 + return await s3AvatarSource.moveAvatarFromTemp(avatar); // 永久化临时的预览头像 + } + })(); + await MongoDataset.findByIdAndUpdate( id, { ...parseParentIdInMongo(parentId), ...(name && { name }), - ...(avatar && { avatar }), + ...(avatar && { avatar: permanentAvatar }), ...(agentModel && { agentModel }), ...(vlmModel && { vlmModel }), ...(websiteConfig && { websiteConfig }), @@ -224,8 +233,6 @@ async function handler( dataset, autoSync }); - - await refreshSourceAvatar(avatar, dataset.avatar, session); }; await mongoSessionRun(async (session) => { diff --git a/projects/app/src/pages/api/support/user/account/update.ts b/projects/app/src/pages/api/support/user/account/update.ts index dfa451e29028..89da6a801f1e 100644 --- a/projects/app/src/pages/api/support/user/account/update.ts +++ b/projects/app/src/pages/api/support/user/account/update.ts @@ -8,6 +8,7 @@ import { NextAPI } from '@/service/middleware/entry'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller'; import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; export type UserAccountUpdateQuery = {}; export type UserAccountUpdateBody = UserUpdateParams; @@ -37,6 +38,10 @@ async function handler( } // if avatar, update team member avatar if (avatar) { + if (tmb?.avatar) { + await getS3AvatarSource().removeAvatar(tmb.avatar); + } + await MongoTeamMember.updateOne( { _id: tmbId @@ -45,7 +50,6 @@ async function handler( avatar } ).session(session); - await refreshSourceAvatar(avatar, tmb?.avatar, session); } }); diff --git a/projects/app/src/pages/api/system/img/[...id].ts b/projects/app/src/pages/api/system/img/[...id].ts new file mode 100644 index 000000000000..90d0df6be51d --- /dev/null +++ b/projects/app/src/pages/api/system/img/[...id].ts @@ -0,0 +1,32 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import path from 'path'; + +import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; +import { Types } from '@fastgpt/service/common/mongo'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; + +// get the models available to the system +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { id } = req.query as { id: string[] }; + + const joined = id.join('/'); + const parsed = path.parse(joined); + const keys = path.format({ dir: parsed.dir, name: parsed.name, ext: '' }); + + if (Types.ObjectId.isValid(keys)) { + const { binary, mime } = await readMongoImg({ id: joined }); + res.setHeader('Content-Type', mime); + res.send(binary); + return; + } + + res.redirect(301, getS3AvatarSource().createPublicUrl(joined)); + } catch (error) { + jsonRes(res, { + code: 500, + error + }); + } +} diff --git a/projects/app/src/pages/api/system/img/[id].ts b/projects/app/src/pages/api/system/img/[id].ts deleted file mode 100644 index 882d21ab3c87..000000000000 --- a/projects/app/src/pages/api/system/img/[id].ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; - -import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; - -// get the models available to the system -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { id } = req.query as { id: string }; - - const { binary, mime } = await readMongoImg({ id }); - - res.setHeader('Content-Type', mime); - res.send(binary); - } catch (error) { - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/service/common/system/cron.ts b/projects/app/src/service/common/system/cron.ts index 70127a39523c..7df8d292922a 100644 --- a/projects/app/src/service/common/system/cron.ts +++ b/projects/app/src/service/common/system/cron.ts @@ -14,6 +14,7 @@ import { getScheduleTriggerApp } from '@/service/core/app/utils'; import { clearExpiredRawTextBufferCron } from '@fastgpt/service/common/buffer/rawText/controller'; import { clearExpiredDatasetImageCron } from '@fastgpt/service/core/dataset/image/controller'; import { cronRefreshModels } from '@fastgpt/service/core/ai/config/utils'; +import { clearExpiredMinioFilesCron } from '@fastgpt/service/common/file/minioTtl/controller'; // Try to run train every minute const setTrainingQueueCron = () => { @@ -90,4 +91,5 @@ export const startCron = () => { clearExpiredRawTextBufferCron(); clearExpiredDatasetImageCron(); cronRefreshModels(); + clearExpiredMinioFilesCron(); }; diff --git a/projects/app/src/web/common/file/api.ts b/projects/app/src/web/common/file/api.ts index c73dad821095..d0e8b96f098e 100644 --- a/projects/app/src/web/common/file/api.ts +++ b/projects/app/src/web/common/file/api.ts @@ -1,6 +1,6 @@ -import { DELETE, GET, POST } from '@/web/common/api/request'; +import { POST } from '@/web/common/api/request'; import type { UploadImgProps } from '@fastgpt/global/common/file/api.d'; -import type { UploadPresignedURLResponse } from '@fastgpt/service/common/s3/type'; +import type { CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; import { type AxiosProgressEvent } from 'axios'; export const postUploadImg = (e: UploadImgProps) => POST('/common/file/uploadImage', e); @@ -32,3 +32,7 @@ export const postS3UploadFile = ( }, onUploadProgress }); + +export const getUploadAvatarPresignedUrl = (params: { filename: string; temporay: boolean }) => { + return POST('/common/file/updateAvatar', params); +}; diff --git a/projects/app/src/web/common/file/utils.ts b/projects/app/src/web/common/file/utils.ts index 160ad29ea3ac..38358902d874 100644 --- a/projects/app/src/web/common/file/utils.ts +++ b/projects/app/src/web/common/file/utils.ts @@ -32,3 +32,15 @@ export const fileToBase64 = (file: File) => { reader.onerror = (error) => reject(error); }); }; + +export const base64ToFile = (base64: string, filename: string) => { + const arr = base64.split(','); + const mime = arr[0].match(/:(.*?);/)?.[1]; + const bstr = atob(arr[1]); + let n = bstr.length; + const u8arr = new Uint8Array(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n); + } + return new File([u8arr], filename, { type: mime }); +};