diff --git a/packages/service/common/s3/buckets/base.ts b/packages/service/common/s3/buckets/base.ts new file mode 100644 index 000000000000..5a9d8c7d248b --- /dev/null +++ b/packages/service/common/s3/buckets/base.ts @@ -0,0 +1,102 @@ +import { Client, type RemoveOptions, type CopyConditions, type LifecycleConfig } from 'minio'; +import { + defaultS3Options, + type CreatePostPresignedUrlOptions, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3BucketName, + type S3Options +} from '../types'; +import type { IBucketBasicOperations } from '../interface'; +import { + createObjectKey, + createPresignedUrlExpires, + createTempObjectKey, + inferContentType +} from '../helpers'; + +export class S3BaseBucket implements IBucketBasicOperations { + public client: Client; + + /** + * + * @param _bucket the bucket you want to operate + * @param options the options for the s3 client + * @param afterInits the function to be called after instantiating the s3 service + */ + constructor( + private readonly _bucket: S3BucketName, + private readonly afterInits?: (() => Promise | void)[], + public options: Partial = defaultS3Options + ) { + options = { ...defaultS3Options, ...options }; + this.options = options as S3Options; + this.client = new Client(options as S3Options); + + const init = async () => { + if (!(await this.exist())) { + await this.client.makeBucket(this._bucket); + } + await Promise.all(this.afterInits?.map((afterInit) => afterInit()) ?? []); + }; + init(); + } + + get name(): string { + return this._bucket; + } + + async move(src: string, dst: string, options?: CopyConditions): Promise { + const bucket = this.name; + await this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options); + return this.client.removeObject(bucket, src); + } + + copy(src: string, dst: string, options?: CopyConditions): ReturnType { + return this.client.copyObject(this.name, src, dst, options); + } + + exist(): Promise { + return this.client.bucketExists(this.name); + } + + delete(objectKey: string, options?: RemoveOptions): Promise { + return this.client.removeObject(this.name, objectKey, options); + } + + get(): Promise { + throw new Error('Method not implemented.'); + } + + lifecycle(): Promise { + return this.client.getBucketLifecycle(this.name); + } + + async createPostPresignedUrl( + params: CreatePostPresignedUrlParams, + options: CreatePostPresignedUrlOptions = {} + ): Promise { + const { temporay } = options; + const contentType = inferContentType(params.filename); + const maxFileSize = this.options.maxFileSize as number; + const key = temporay ? createTempObjectKey(params) : createObjectKey(params); + + const policy = this.client.newPostPolicy(); + policy.setKey(key); + policy.setBucket(this.name); + policy.setContentType(contentType); + policy.setContentLengthRange(1, maxFileSize); + policy.setExpires(createPresignedUrlExpires(10)); + policy.setUserMetaData({ + filename: encodeURIComponent(params.filename), + visibility: params.visibility + }); + + const { formData, postURL } = await this.client.presignedPostPolicy(policy); + + return { + url: postURL, + fields: formData + }; + } +} diff --git a/packages/service/common/s3/buckets/manager.ts b/packages/service/common/s3/buckets/manager.ts new file mode 100644 index 000000000000..b3f40d0a6c69 --- /dev/null +++ b/packages/service/common/s3/buckets/manager.ts @@ -0,0 +1,24 @@ +import { type S3BaseBucket } from './base'; +import { type S3Options } from '../types'; +import { S3PublicBucket } from './public'; +import { S3PrivateBucket } from './private'; + +export class S3BucketManager { + private static instance: S3BucketManager; + private publicBucket: S3PublicBucket | null = null; + private privateBucket: S3PrivateBucket | null = null; + + private constructor() {} + + static getInstance(): S3BucketManager { + return (this.instance ??= new S3BucketManager()); + } + + getPublicBucket(options?: Partial): S3PublicBucket { + return (this.publicBucket ??= new S3PublicBucket(options)); + } + + getPrivateBucket(options?: Partial): S3PrivateBucket { + return (this.privateBucket ??= new S3PrivateBucket(options)); + } +} diff --git a/packages/service/common/s3/buckets/private.ts b/packages/service/common/s3/buckets/private.ts new file mode 100644 index 000000000000..380ac1503b50 --- /dev/null +++ b/packages/service/common/s3/buckets/private.ts @@ -0,0 +1,19 @@ +import { S3BaseBucket } from './base'; +import { + S3Buckets, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; + +export class S3PrivateBucket extends S3BaseBucket { + constructor(options?: Partial) { + super(S3Buckets.private, undefined, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return super.createPostPresignedUrl({ ...params, visibility: 'private' }); + } +} diff --git a/packages/service/common/s3/buckets/public.ts b/packages/service/common/s3/buckets/public.ts new file mode 100644 index 000000000000..4843c910dc94 --- /dev/null +++ b/packages/service/common/s3/buckets/public.ts @@ -0,0 +1,54 @@ +import { S3BaseBucket } from './base'; +import { createBucketPolicy } from '../helpers'; +import { + S3Buckets, + type CreatePostPresignedUrlOptions, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { IPublicBucketOperations } from '../interface'; +import { lifecycleOfTemporaryAvatars } from '../lifecycle'; + +export class S3PublicBucket extends S3BaseBucket implements IPublicBucketOperations { + constructor(options?: Partial) { + super( + S3Buckets.public, + [ + // set bucket policy + async () => { + const bucket = this.name; + const policy = createBucketPolicy(bucket); + try { + await this.client.setBucketPolicy(bucket, policy); + } catch (error) { + // TODO: maybe it was a cloud S3 that doesn't allow us to set the policy, so that cause the error, + // maybe we can ignore the error, or we have other plan to handle this. + } + }, + // set bucket lifecycle + async () => { + const bucket = this.name; + await this.client.setBucketLifecycle(bucket, lifecycleOfTemporaryAvatars); + } + ], + options + ); + } + + createPublicUrl(objectKey: string): string { + const protocol = this.options.useSSL ? 'https' : 'http'; + const hostname = this.options.endPoint; + const port = this.options.port; + const bucket = this.name; + + return `${protocol}://${hostname}:${port}/${bucket}/${objectKey}`; + } + + override createPostPresignedUrl( + params: Omit, + options: CreatePostPresignedUrlOptions = {} + ): Promise { + return super.createPostPresignedUrl({ ...params, visibility: 'public' }, options); + } +} diff --git a/packages/service/common/s3/config.ts b/packages/service/common/s3/config.ts deleted file mode 100644 index 501d61be4b1a..000000000000 --- a/packages/service/common/s3/config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { S3ServiceConfig } from './type'; - -export const defualtS3Config: Omit = { - endPoint: process.env.S3_ENDPOINT || 'localhost', - port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, - useSSL: process.env.S3_USE_SSL === 'true', - accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', - secretKey: process.env.S3_SECRET_KEY || 'minioadmin', - externalBaseURL: process.env.S3_EXTERNAL_BASE_URL -}; diff --git a/packages/service/common/s3/const.ts b/packages/service/common/s3/const.ts deleted file mode 100644 index c91043dc1f96..000000000000 --- a/packages/service/common/s3/const.ts +++ /dev/null @@ -1,20 +0,0 @@ -export const mimeMap: Record = { - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.png': 'image/png', - '.gif': 'image/gif', - '.webp': 'image/webp', - '.svg': 'image/svg+xml', - '.pdf': 'application/pdf', - '.txt': 'text/plain', - '.json': 'application/json', - '.csv': 'text/csv', - '.zip': 'application/zip', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - '.doc': 'application/msword', - '.xls': 'application/vnd.ms-excel', - '.ppt': 'application/vnd.ms-powerpoint', - '.js': 'application/javascript' -}; diff --git a/packages/service/common/s3/controller.ts b/packages/service/common/s3/controller.ts deleted file mode 100644 index faaae7cdc91e..000000000000 --- a/packages/service/common/s3/controller.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { Client } from 'minio'; -import { - type FileMetadataType, - type PresignedUrlInput as UploadPresignedURLProps, - type UploadPresignedURLResponse, - type S3ServiceConfig -} from './type'; -import { defualtS3Config } from './config'; -import { randomBytes } from 'crypto'; -import { HttpProxyAgent } from 'http-proxy-agent'; -import { HttpsProxyAgent } from 'https-proxy-agent'; -import { extname } from 'path'; -import { addLog } from '../../common/system/log'; -import { getErrText } from '@fastgpt/global/common/error/utils'; -import { mimeMap } from './const'; - -export class S3Service { - private client: Client; - private config: S3ServiceConfig; - private initialized: boolean = false; - initFunction?: () => Promise; - - constructor(config?: Partial) { - this.config = { ...defualtS3Config, ...config } as S3ServiceConfig; - - this.client = new Client({ - endPoint: this.config.endPoint, - port: this.config.port, - useSSL: this.config.useSSL, - accessKey: this.config.accessKey, - secretKey: this.config.secretKey, - transportAgent: process.env.HTTP_PROXY - ? new HttpProxyAgent(process.env.HTTP_PROXY) - : process.env.HTTPS_PROXY - ? new HttpsProxyAgent(process.env.HTTPS_PROXY) - : undefined - }); - - this.initFunction = config?.initFunction; - } - - public async init() { - if (!this.initialized) { - if (!(await this.client.bucketExists(this.config.bucket))) { - addLog.debug(`Creating bucket: ${this.config.bucket}`); - await this.client.makeBucket(this.config.bucket); - } - - await this.initFunction?.(); - this.initialized = true; - } - } - - private generateFileId(): string { - return randomBytes(16).toString('hex'); - } - - private generateAccessUrl(filename: string): string { - const protocol = this.config.useSSL ? 'https' : 'http'; - const port = - this.config.port && this.config.port !== (this.config.useSSL ? 443 : 80) - ? `:${this.config.port}` - : ''; - - const externalBaseURL = this.config.externalBaseURL; - return externalBaseURL - ? `${externalBaseURL}/${this.config.bucket}/${encodeURIComponent(filename)}` - : `${protocol}://${this.config.endPoint}${port}/${this.config.bucket}/${encodeURIComponent(filename)}`; - } - - uploadFile = async (fileBuffer: Buffer, originalFilename: string): Promise => { - await this.init(); - const inferContentType = (filename: string) => { - const ext = extname(filename).toLowerCase(); - return mimeMap[ext] || 'application/octet-stream'; - }; - - if (this.config.maxFileSize && fileBuffer.length > this.config.maxFileSize) { - return Promise.reject( - `File size ${fileBuffer.length} exceeds limit ${this.config.maxFileSize}` - ); - } - - const fileId = this.generateFileId(); - const objectName = `${fileId}-${originalFilename}`; - const uploadTime = new Date(); - - const contentType = inferContentType(originalFilename); - await this.client.putObject(this.config.bucket, objectName, fileBuffer, fileBuffer.length, { - 'Content-Type': contentType, - 'Content-Disposition': `attachment; filename="${encodeURIComponent(originalFilename)}"`, - 'x-amz-meta-original-filename': encodeURIComponent(originalFilename), - 'x-amz-meta-upload-time': uploadTime.toISOString() - }); - - const metadata: FileMetadataType = { - fileId, - originalFilename, - contentType, - size: fileBuffer.length, - uploadTime, - accessUrl: this.generateAccessUrl(objectName) - }; - - return metadata; - }; - - generateUploadPresignedURL = async ({ - filepath, - contentType, - metadata, - filename - }: UploadPresignedURLProps): Promise => { - await this.init(); - const objectName = `${filepath}/${filename}`; - - try { - const policy = this.client.newPostPolicy(); - - policy.setBucket(this.config.bucket); - policy.setKey(objectName); - if (contentType) { - policy.setContentType(contentType); - } - if (this.config.maxFileSize) { - policy.setContentLengthRange(1, this.config.maxFileSize); - } - policy.setExpires(new Date(Date.now() + 10 * 60 * 1000)); // 10 mins - - policy.setUserMetaData({ - 'original-filename': encodeURIComponent(filename), - 'upload-time': new Date().toISOString(), - ...metadata - }); - - const { postURL, formData } = await this.client.presignedPostPolicy(policy); - - const response: UploadPresignedURLResponse = { - objectName, - uploadUrl: postURL, - formData - }; - - return response; - } catch (error) { - addLog.error('Failed to generate Upload Presigned URL', error); - return Promise.reject(`Failed to generate Upload Presigned URL: ${getErrText(error)}`); - } - }; - - generateDownloadUrl = (objectName: string): string => { - const pathParts = objectName.split('/'); - const encodedParts = pathParts.map((part) => encodeURIComponent(part)); - const encodedObjectName = encodedParts.join('/'); - return `${this.config.bucket}/${encodedObjectName}`; - }; - - getFile = async (objectName: string): Promise => { - const stat = await this.client.statObject(this.config.bucket, objectName); - - if (stat.size > 0) { - const accessUrl = this.generateDownloadUrl(objectName); - return accessUrl; - } - - return Promise.reject(`File ${objectName} not found`); - }; -} diff --git a/packages/service/common/s3/helpers.ts b/packages/service/common/s3/helpers.ts new file mode 100644 index 000000000000..2b230a3f5342 --- /dev/null +++ b/packages/service/common/s3/helpers.ts @@ -0,0 +1,79 @@ +import path from 'node:path'; +import crypto from 'node:crypto'; +import { type ContentType, type CreateObjectKeyParams, type ExtensionType, Mimes } from './types'; +import dayjs from 'dayjs'; + +/** + * + * @param filename + * @returns the Content-Type relative to the mime type + */ +export const inferContentType = (filename: string): ContentType | 'application/octet-stream' => { + const ext = path.extname(filename).toLowerCase() as ExtensionType; + return Mimes[ext] ?? 'application/octet-stream'; +}; + +/** + * Generate a date that is `minutes` minutes from now + * + * @param minutes + * @returns the date object + */ +export const createPresignedUrlExpires = (minutes: number): Date => { + return new Date(Date.now() + minutes * 60 * 1_000); +}; + +/** + * use public policy or just a custom policy + * + * @default policy public policy + * @param bucket bucket name + * @returns the policy string + */ +export const createBucketPolicy = ( + bucket: string, + policy?: 'public' | Record +): string => { + if (typeof policy === 'string' && policy !== 'public') { + throw new Error("'policy' only can be assigned to 'public' if typeof 'policy' is string"); + } + + switch (typeof policy) { + case 'string': + case 'undefined': + default: + return JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: '*', + Action: 's3:GetObject', + Resource: `arn:aws:s3:::${bucket}/*` + } + ] + }); + case 'object': + if (policy === null) { + throw new Error("Bucket policy can't be null"); + } + return JSON.stringify(policy); + } +}; + +/** + * create s3 object key by source, team ID and filename + */ +export function createObjectKey({ source, teamId, filename }: CreateObjectKeyParams): string { + const date = dayjs().format('YYYY_MM_DD'); + const id = crypto.randomBytes(16).toString('hex'); + return `${source}/${teamId}/${date}/${id}_${filename}`; +} + +/** + * create temporary s3 object key by source, team ID and filename + */ +export function createTempObjectKey(params: CreateObjectKeyParams): string { + const origin = createObjectKey(params); + return `temp/${origin}`; +} diff --git a/packages/service/common/s3/index.ts b/packages/service/common/s3/index.ts deleted file mode 100644 index 761bd564097e..000000000000 --- a/packages/service/common/s3/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { S3Service } from './controller'; - -export const PluginS3Service = (() => { - if (!global.pluginS3Service) { - global.pluginS3Service = new S3Service({ - bucket: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', - maxFileSize: 50 * 1024 * 1024 // 50MB - }); - } - - return global.pluginS3Service; -})(); - -declare global { - var pluginS3Service: S3Service; -} diff --git a/packages/service/common/s3/interface.ts b/packages/service/common/s3/interface.ts new file mode 100644 index 000000000000..3f99a6fb4c92 --- /dev/null +++ b/packages/service/common/s3/interface.ts @@ -0,0 +1,20 @@ +import { type LifecycleConfig, type Client, type CopyConditions, type RemoveOptions } from 'minio'; +import type { CreateObjectKeyParams, CreatePostPresignedUrlOptions } from './types'; + +export interface IBucketBasicOperations { + get name(): string; + get(): Promise; + exist(): Promise; + delete(objectKey: string, options?: RemoveOptions): Promise; + move(src: string, dst: string, options?: CopyConditions): Promise; + copy(src: string, dst: string, options?: CopyConditions): ReturnType; + lifecycle(): Promise; + createPostPresignedUrl( + params: CreateObjectKeyParams, + options?: CreatePostPresignedUrlOptions + ): Promise<{ url: string; fields: Record }>; +} + +export interface IPublicBucketOperations { + createPublicUrl(objectKey: string): string; +} diff --git a/packages/service/common/s3/lifecycle.ts b/packages/service/common/s3/lifecycle.ts new file mode 100644 index 000000000000..12c28c1ca746 --- /dev/null +++ b/packages/service/common/s3/lifecycle.ts @@ -0,0 +1,23 @@ +import { type LifecycleRule, type LifecycleConfig } from 'minio'; + +export function createLifeCycleConfig(rule: LifecycleRule): LifecycleConfig { + return { + Rule: [rule] + }; +} + +export function assembleLifeCycleConfigs(...configs: LifecycleConfig[]): LifecycleConfig { + if (configs.length === 0) return { Rule: [] }; + return { + Rule: configs.flatMap((config) => config.Rule) + }; +} + +export const lifecycleOfTemporaryAvatars = createLifeCycleConfig({ + ID: 'Temporary Avatars Rule', + Prefix: 'temp/avatar/', + Status: 'Enabled', + Expiration: { + Days: 1 + } +}); diff --git a/packages/service/common/s3/sources/avatar.ts b/packages/service/common/s3/sources/avatar.ts new file mode 100644 index 000000000000..f71d29e472c9 --- /dev/null +++ b/packages/service/common/s3/sources/avatar.ts @@ -0,0 +1,57 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + S3APIPrefix, + type CreatePostPresignedUrlOptions, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PublicBucket } from '../buckets/public'; + +class S3AvatarSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.avatar, true, options); + } + + static getInstance(options?: Partial): S3AvatarSource { + return S3BaseSource._getInstance(S3AvatarSource, options); + } + + override createPostPresignedUrl( + params: Omit, + options: CreatePostPresignedUrlOptions = {} + ): Promise { + return this.bucket.createPostPresignedUrl( + { + ...params, + source: S3Sources.avatar + }, + options + ); + } + + createPublicUrl(objectKey: string): string { + return this.bucket.createPublicUrl(objectKey); + } + + createAvatarObjectKey(avatarWithPrefix: string): string { + return avatarWithPrefix.replace(S3APIPrefix.avatar, ''); + } + + removeAvatar(avatarWithPrefix: string): Promise { + const avatarObjectKey = this.createAvatarObjectKey(avatarWithPrefix); + return this.bucket.delete(avatarObjectKey); + } + + async moveAvatarFromTemp(tempAvatarWithPrefix: string): Promise { + const tempAvatarObjectKey = this.createAvatarObjectKey(tempAvatarWithPrefix); + const avatarObjectKey = tempAvatarObjectKey.replace(`${S3Sources.temp}/`, ''); + await this.bucket.move(tempAvatarObjectKey, avatarObjectKey); + return S3APIPrefix.avatar + avatarObjectKey; + } +} + +export function getS3AvatarSource() { + return S3AvatarSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/base.ts b/packages/service/common/s3/sources/base.ts new file mode 100644 index 000000000000..a34dd20ffece --- /dev/null +++ b/packages/service/common/s3/sources/base.ts @@ -0,0 +1,58 @@ +import { S3BucketManager } from '../buckets/manager'; +import type { S3PrivateBucket } from '../buckets/private'; +import type { S3PublicBucket } from '../buckets/public'; +import type { + CreatePostPresignedUrlParams, + CreatePostPresignedUrlResult, + S3Options, + S3SourceType +} from '../types'; + +type Bucket = S3PublicBucket | S3PrivateBucket; + +export abstract class S3BaseSource { + protected bucket: T; + protected static instances: Map = new Map(); + + constructor( + protected readonly source: S3SourceType, + protected readonly pub: boolean = false, + options?: Partial + ) { + const manager = S3BucketManager.getInstance(); + switch (pub) { + case true: + this.bucket = manager.getPublicBucket(options) as T; + break; + case false: + this.bucket = manager.getPrivateBucket(options) as T; + } + } + + abstract createPostPresignedUrl( + params: Omit + ): Promise; + + protected static _getInstance( + constructor: new (options?: Partial) => T, + options?: Partial + ): T { + const className = constructor.name; + if (!S3BaseSource.instances.has(className)) { + S3BaseSource.instances.set(className, new constructor(options)); + } + return S3BaseSource.instances.get(className) as T; + } + + protected getBucket(): T { + return this.bucket; + } + + get bucketName(): string { + return this.bucket.name; + } + + isBucketExist(): Promise { + return this.bucket.exist(); + } +} diff --git a/packages/service/common/s3/sources/chat.ts b/packages/service/common/s3/sources/chat.ts new file mode 100644 index 000000000000..13094173d23e --- /dev/null +++ b/packages/service/common/s3/sources/chat.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3ChatSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.chat, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.chat }); + } + + static getInstance(options?: Partial): S3ChatSource { + return S3BaseSource._getInstance(S3ChatSource, options); + } +} + +export function getS3ChatSource() { + return S3ChatSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/dataset-image.ts b/packages/service/common/s3/sources/dataset-image.ts new file mode 100644 index 000000000000..11c6951b589d --- /dev/null +++ b/packages/service/common/s3/sources/dataset-image.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3DatasetImageSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources['dataset-image'], false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources['dataset-image'] }); + } + + static getInstance(options?: Partial): S3DatasetImageSource { + return S3BaseSource._getInstance(S3DatasetImageSource, options); + } +} + +export function getS3DatasetImageSource() { + return S3DatasetImageSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/dataset.ts b/packages/service/common/s3/sources/dataset.ts new file mode 100644 index 000000000000..58aa507823c3 --- /dev/null +++ b/packages/service/common/s3/sources/dataset.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3DatasetSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.dataset, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.dataset }); + } + + static getInstance(options?: Partial): S3DatasetSource { + return S3BaseSource._getInstance(S3DatasetSource, options); + } +} + +export function getS3DatasetSource() { + return S3DatasetSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/index.ts b/packages/service/common/s3/sources/index.ts new file mode 100644 index 000000000000..a19622639276 --- /dev/null +++ b/packages/service/common/s3/sources/index.ts @@ -0,0 +1,24 @@ +import { getS3AvatarSource } from './avatar'; +import { getS3ChatSource } from './chat'; +import { getS3DatasetSource } from './dataset'; +import { getS3DatasetImageSource } from './dataset-image'; +import { getS3InvoiceSource } from './invoice'; +import { getS3RawtextSource } from './rawtext'; + +export function registerSources() { + getS3AvatarSource(); + getS3ChatSource(); + getS3DatasetImageSource(); + getS3DatasetSource(); + getS3InvoiceSource(); + getS3RawtextSource(); +} + +export { + getS3AvatarSource, + getS3ChatSource, + getS3DatasetImageSource, + getS3DatasetSource, + getS3InvoiceSource, + getS3RawtextSource +}; diff --git a/packages/service/common/s3/sources/invoice.ts b/packages/service/common/s3/sources/invoice.ts new file mode 100644 index 000000000000..453020a122a5 --- /dev/null +++ b/packages/service/common/s3/sources/invoice.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3InvoiceSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.invoice, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.invoice }); + } + + static getInstance(options?: Partial): S3InvoiceSource { + return S3BaseSource._getInstance(S3InvoiceSource, options); + } +} + +export function getS3InvoiceSource() { + return S3InvoiceSource.getInstance(); +} diff --git a/packages/service/common/s3/sources/rawtext.ts b/packages/service/common/s3/sources/rawtext.ts new file mode 100644 index 000000000000..d96792003b34 --- /dev/null +++ b/packages/service/common/s3/sources/rawtext.ts @@ -0,0 +1,28 @@ +import { S3BaseSource } from './base'; +import { + S3Sources, + type CreatePostPresignedUrlParams, + type CreatePostPresignedUrlResult, + type S3Options +} from '../types'; +import type { S3PrivateBucket } from '../buckets/private'; + +class S3RawtextSource extends S3BaseSource { + constructor(options?: Partial) { + super(S3Sources.rawtext, false, options); + } + + override createPostPresignedUrl( + params: Omit + ): Promise { + return this.bucket.createPostPresignedUrl({ ...params, source: S3Sources.rawtext }); + } + + static getInstance(options?: Partial): S3RawtextSource { + return S3BaseSource._getInstance(S3RawtextSource, options); + } +} + +export function getS3RawtextSource() { + return S3RawtextSource.getInstance(); +} diff --git a/packages/service/common/s3/type.ts b/packages/service/common/s3/type.ts deleted file mode 100644 index a480530493e8..000000000000 --- a/packages/service/common/s3/type.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { ClientOptions } from 'minio'; - -export type S3ServiceConfig = { - bucket: string; - externalBaseURL?: string; - /** - * Unit: Byte - */ - maxFileSize?: number; - /** - * for executing some init function for the s3 service - */ - initFunction?: () => Promise; -} & ClientOptions; - -export type FileMetadataType = { - fileId: string; - originalFilename: string; - contentType: string; - size: number; - uploadTime: Date; - accessUrl: string; -}; - -export type PresignedUrlInput = { - filepath: string; - filename: string; - contentType?: string; - metadata?: Record; -}; - -export type UploadPresignedURLResponse = { - objectName: string; - uploadUrl: string; - formData: Record; -}; - -export type FileUploadInput = { - buffer: Buffer; - filename: string; -}; - -export enum PluginTypeEnum { - tool = 'tool' -} - -export const PluginFilePath = { - [PluginTypeEnum.tool]: 'plugin/tools' -}; diff --git a/packages/service/common/s3/types.ts b/packages/service/common/s3/types.ts new file mode 100644 index 000000000000..d2be9c357448 --- /dev/null +++ b/packages/service/common/s3/types.ts @@ -0,0 +1,105 @@ +import type { ClientOptions } from 'minio'; +import { HttpProxyAgent } from 'http-proxy-agent'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import { z } from 'zod'; + +export const S3MetadataSchema = z.object({ + filename: z.string(), + uploadedAt: z.date(), + accessUrl: z.string(), + contentType: z.string(), + id: z.string().length(32), + size: z.number().positive() +}); +export type S3Metadata = z.infer; + +export const Mimes = { + '.gif': 'image/gif', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + + '.csv': 'text/csv', + '.txt': 'text/plain', + + '.pdf': 'application/pdf', + '.zip': 'application/zip', + '.json': 'application/json', + '.doc': 'application/msword', + '.js': 'application/javascript', + '.xls': 'application/vnd.ms-excel', + '.ppt': 'application/vnd.ms-powerpoint', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation' +} as const; +export type ContentType = (typeof Mimes)[keyof typeof Mimes]; +export type ExtensionType = keyof typeof Mimes; + +export const defaultS3Options: { externalBaseURL?: string; maxFileSize?: number } & ClientOptions = + { + maxFileSize: 1024 ** 3, // 1GB + + useSSL: process.env.S3_USE_SSL === 'true', + endPoint: process.env.S3_ENDPOINT || 'localhost', + externalBaseURL: process.env.S3_EXTERNAL_BASE_URL, + accessKey: process.env.S3_ACCESS_KEY || 'minioadmin', + secretKey: process.env.S3_SECRET_KEY || 'minioadmin', + port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : 9000, + transportAgent: process.env.HTTP_PROXY + ? new HttpProxyAgent(process.env.HTTP_PROXY) + : process.env.HTTPS_PROXY + ? new HttpsProxyAgent(process.env.HTTPS_PROXY) + : undefined + }; +export type S3Options = typeof defaultS3Options; + +export const S3Buckets = { + plugin: process.env.S3_PLUGIN_BUCKET || 'fastgpt-plugin', + public: process.env.S3_PUBLIC_BUCKET || 'fastgpt-public', + private: process.env.S3_PRIVATE_BUCKET || 'fastgpt-private' +} as const; +export type S3BucketName = (typeof S3Buckets)[keyof typeof S3Buckets]; + +export const S3SourcesSchema = z.enum([ + 'avatar', + 'chat', + 'dataset', + 'dataset-image', + 'invoice', + 'rawtext', + 'temp' +]); +export const S3Sources = S3SourcesSchema.enum; +export type S3SourceType = z.infer; + +export const CreateObjectKeyParamsSchema = z.object({ + filename: z.string().min(1), + source: S3SourcesSchema, + teamId: z.string().length(16) +}); +export type CreateObjectKeyParams = z.infer; + +export const CreatePostPresignedUrlParamsSchema = z.object({ + ...CreateObjectKeyParamsSchema.shape, + visibility: z.enum(['public', 'private']).default('private') +}); +export type CreatePostPresignedUrlParams = z.infer; + +export const CreatePostPresignedUrlOptionsSchema = z.object({ + temporay: z.boolean().optional() +}); +export type CreatePostPresignedUrlOptions = z.infer; + +export const CreatePostPresignedUrlResultSchema = z.object({ + url: z.string().min(1), + fields: z.record(z.string(), z.string()) +}); +export type CreatePostPresignedUrlResult = z.infer; + +export const S3APIPrefix = { + avatar: '/api/system/img/' +} as const; +export type S3APIPrefixType = (typeof S3APIPrefix)[keyof typeof S3APIPrefix]; diff --git a/packages/service/package.json b/packages/service/package.json index eb367ff8ed38..41438d10f7af 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -54,7 +54,8 @@ "tiktoken": "1.0.17", "tunnel": "^0.0.6", "turndown": "^7.1.2", - "winston": "^3.17.0" + "winston": "^3.17.0", + "zod": "^3.24.2" }, "devDependencies": { "@types/cookie": "^0.5.2", diff --git a/packages/web/common/file/hooks/useUploadAvatar.tsx b/packages/web/common/file/hooks/useUploadAvatar.tsx new file mode 100644 index 000000000000..10bf1ef04897 --- /dev/null +++ b/packages/web/common/file/hooks/useUploadAvatar.tsx @@ -0,0 +1,91 @@ +import { base64ToFile, fileToBase64 } from '../utils'; +import { compressBase64Img } from '../img'; +import { useToast } from '../../../hooks/useToast'; +import { useCallback, useRef, useTransition } from 'react'; +import { useTranslation } from 'next-i18next'; +import { type CreatePostPresignedUrlResult } from '../../../../service/common/s3/types'; + +export const useUploadAvatar = ( + api: (params: { filename: string; temporay: boolean }) => Promise, + { + temporay = false, + onSuccess + }: { + temporay?: boolean; + onSuccess?: (avatar: string) => void; + } = {} +) => { + const { toast } = useToast(); + const { t } = useTranslation(); + const [uploading, startUpload] = useTransition(); + const uploadAvatarRef = useRef(null); + + const handleFileSelectorOpen = useCallback(() => { + if (!uploadAvatarRef.current) return; + uploadAvatarRef.current.click(); + }, []); + + const onUploadAvatarChange = useCallback( + async (e: React.ChangeEvent) => { + const files = e.target.files; + + if (!files || files.length === 0) { + e.target.value = ''; + return; + } + if (files.length > 1) { + toast({ title: t('account_info:avatar_can_only_select_one'), status: 'warning' }); + e.target.value = ''; + return; + } + const file = files[0]!; + + if (!file.name.match(/\.(jpg|png|jpeg)$/)) { + toast({ title: t('account_info:avatar_can_only_select_jpg_png'), status: 'warning' }); + e.target.value = ''; + return; + } + + startUpload(async () => { + const compressed = base64ToFile( + await compressBase64Img({ + base64Img: await fileToBase64(file), + maxW: 300, + maxH: 300 + }), + file.name + ); + const { url, fields } = await api({ filename: file.name, temporay }); + const formData = new FormData(); + Object.entries(fields).forEach(([k, v]) => formData.set(k, v)); + formData.set('file', compressed); + await fetch(url, { method: 'POST', body: formData }); // 204 + + const avatar = `/api/system/img/${fields.key}`; + onSuccess?.(avatar); + + e.target.value = ''; + }); + }, + [t, temporay, toast, onSuccess, api] + ); + + const Component = useCallback(() => { + return ( + + ); + }, [onUploadAvatarChange]); + + return { + uploading, + Component, + handleFileSelectorOpen + }; +}; diff --git a/packages/web/common/file/utils.ts b/packages/web/common/file/utils.ts index 7fd10744530f..f4107301d585 100644 --- a/packages/web/common/file/utils.ts +++ b/packages/web/common/file/utils.ts @@ -99,3 +99,24 @@ async function detectFileEncoding(file: File): Promise { return encoding || 'utf-8'; } + +export const fileToBase64 = (file: File) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => resolve(reader.result as string); + reader.onerror = (error) => reject(error); + }); +}; + +export const base64ToFile = (base64: string, filename: string) => { + const arr = base64.split(','); + const mime = arr[0].match(/:(.*?);/)?.[1]; + const bstr = atob(arr[1]); + let n = bstr.length; + const u8arr = new Uint8Array(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n); + } + return new File([u8arr], filename, { type: mime }); +}; diff --git a/packages/web/i18n/en/account_info.json b/packages/web/i18n/en/account_info.json index 15a520732bef..c420e057d285 100644 --- a/packages/web/i18n/en/account_info.json +++ b/packages/web/i18n/en/account_info.json @@ -9,6 +9,8 @@ "app_amount": "App amount", "avatar": "Avatar", "avatar_selection_exception": "Abnormal avatar selection", + "avatar_can_only_select_one": "Avatar can only select one picture", + "avatar_can_only_select_jpg_png": "Avatar can only select jpg or png format", "balance": "balance", "billing_standard": "Standards", "cancel": "Cancel", diff --git a/packages/web/i18n/zh-CN/account_info.json b/packages/web/i18n/zh-CN/account_info.json index cd20076cdcc6..7822d8765995 100644 --- a/packages/web/i18n/zh-CN/account_info.json +++ b/packages/web/i18n/zh-CN/account_info.json @@ -9,6 +9,8 @@ "app_amount": "应用数量", "avatar": "头像", "avatar_selection_exception": "头像选择异常", + "avatar_can_only_select_one": "头像只能选择一张图片", + "avatar_can_only_select_jpg_png": "头像只能选择 jpg 或 png 格式", "balance": "余额", "billing_standard": "计费标准", "cancel": "取消", diff --git a/packages/web/i18n/zh-Hant/account_info.json b/packages/web/i18n/zh-Hant/account_info.json index 9c5560d48d91..5646df649802 100644 --- a/packages/web/i18n/zh-Hant/account_info.json +++ b/packages/web/i18n/zh-Hant/account_info.json @@ -9,6 +9,8 @@ "app_amount": "應用數量", "avatar": "頭像", "avatar_selection_exception": "頭像選擇異常", + "avatar_can_only_select_one": "頭像只能選擇一張圖片", + "avatar_can_only_select_jpg_png": "頭像只能選擇 jpg 或 png 格式", "balance": "餘額", "billing_standard": "計費標準", "cancel": "取消", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7da780021d45..066581a4aea0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -282,6 +282,9 @@ importers: winston: specifier: ^3.17.0 version: 3.17.0 + zod: + specifier: ^3.24.2 + version: 3.25.51 devDependencies: '@types/cookie': specifier: ^0.5.2 @@ -12231,8 +12234,8 @@ snapshots: express-rate-limit: 7.5.0(express@5.1.0) pkce-challenge: 5.0.0 raw-body: 3.0.0 - zod: 3.24.2 - zod-to-json-schema: 3.24.5(zod@3.24.2) + zod: 3.25.51 + zod-to-json-schema: 3.24.5(zod@3.25.51) transitivePeerDependencies: - supports-color @@ -21900,9 +21903,9 @@ snapshots: - terser - typescript - zod-to-json-schema@3.24.5(zod@3.24.2): + zod-to-json-schema@3.24.5(zod@3.25.51): dependencies: - zod: 3.24.2 + zod: 3.25.51 zod-validation-error@3.4.0(zod@3.25.51): dependencies: diff --git a/projects/app/.env.template b/projects/app/.env.template index 308209973914..d9d383ebc66f 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -41,7 +41,10 @@ S3_PORT=9000 S3_USE_SSL=false S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin -S3_PLUGIN_BUCKET=fastgpt-plugin # 插件文件存储bucket +S3_PLUGIN_BUCKET=fastgpt-plugins # 插件文件存储bucket +S3_PUBLIC_BUCKET=fastgpt-public # 插件文件存储公开桶 +S3_PRIVATE_BUCKET=fastgpt-private # 插件文件存储公开桶 + # Redis URL REDIS_URL=redis://default:mypassword@127.0.0.1:6379 # mongo 数据库连接参数，本地开发连接远程数据库时，可能需要增加 directConnection=true 参数，才能连接上。 diff --git a/projects/app/src/components/common/Modal/EditResourceModal.tsx b/projects/app/src/components/common/Modal/EditResourceModal.tsx index 85c8d06d2c49..9827e2bdadc1 100644 --- a/projects/app/src/components/common/Modal/EditResourceModal.tsx +++ b/projects/app/src/components/common/Modal/EditResourceModal.tsx @@ -1,13 +1,14 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { ModalFooter, ModalBody, Input, Button, Box, Textarea, HStack } from '@chakra-ui/react'; import MyModal from '@fastgpt/web/components/common/MyModal/index'; import { useTranslation } from 'next-i18next'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; import { useForm } from 'react-hook-form'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import Avatar from '@fastgpt/web/components/common/Avatar'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; export type EditResourceInfoFormType = { id: string; @@ -41,14 +42,20 @@ const EditResourceModal = ({ } ); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false - }); + const afterUploadAvatar = useCallback( + (avatar: string) => { + console.log('avatar', avatar); + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + temporay: true, + onSuccess: afterUploadAvatar + } + ); return ( @@ -64,7 +71,7 @@ const EditResourceModal = ({ h={'2rem'} cursor={'pointer'} borderRadius={'sm'} - onClick={onOpenSelectFile} + onClick={handleFileSelectorOpen} /> - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> + ); }; diff --git a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx index db6f4574b403..0e0a0e95a114 100644 --- a/projects/app/src/pageComponents/account/team/EditInfoModal.tsx +++ b/projects/app/src/pageComponents/account/team/EditInfoModal.tsx @@ -1,7 +1,6 @@ -import React from 'react'; +import React, { useCallback } from 'react'; import { useForm } from 'react-hook-form'; import { useTranslation } from 'next-i18next'; -import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { @@ -21,6 +20,8 @@ import { type CreateTeamProps } from '@fastgpt/global/support/user/team/controll import { DEFAULT_TEAM_AVATAR } from '@fastgpt/global/common/system/constants'; import Icon from '@fastgpt/web/components/common/Icon'; import dynamic from 'next/dynamic'; +import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar'; +import { getUploadAvatarPresignedUrl } from '@/web/common/file/api'; const UpdateContact = dynamic(() => import('@/components/support/user/inform/UpdateContactModal')); export type EditTeamFormDataType = CreateTeamProps & { @@ -50,15 +51,6 @@ function EditModal({ const avatar = watch('avatar'); const notificationAccount = watch('notificationAccount'); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png,.svg', - multiple: false - }); - const { mutate: onclickCreate, isLoading: creating } = useRequest({ mutationFn: async (data: CreateTeamProps) => { return postCreateTeam(data); @@ -88,6 +80,19 @@ function EditModal({ const { isOpen: isOpenContact, onClose: onCloseContact, onOpen: onOpenContact } = useDisclosure(); + const afterUploadAvatar = useCallback( + (avatar: string) => { + setValue('avatar', avatar); + }, + [setValue] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); + return ( {t('account_team:set_name_avatar')} + )} - - onSelectImage(e, { - maxH: 300, - maxW: 300, - callback: (e) => setValue('avatar', e) - }) - } - /> {isOpenContact && ( import('@/pageComponents/account/info/RedeemCouponModal'), { ssr: false @@ -140,14 +142,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { onClose: onCloseUpdateContact, onOpen: onOpenUpdateContact } = useDisclosure(); - const { - File, - onOpen: onOpenSelectFile, - onSelectImage - } = useSelectFile({ - fileType: '.jpg,.png', - multiple: false - }); const onclickSave = useCallback( async (data: UserType) => { @@ -164,6 +158,20 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { [reset, t, toast, updateUserInfo] ); + const afterUploadAvatar = useCallback( + (avatar: string) => { + if (!userInfo) return; + onclickSave({ ...userInfo, avatar }); + }, + [onclickSave, userInfo] + ); + const { Component: AvatarUploader, handleFileSelectorOpen } = useUploadAvatar( + getUploadAvatarPresignedUrl, + { + onSuccess: afterUploadAvatar + } + ); + const labelStyles: BoxProps = { flex: '0 0 80px', color: 'var(--light-general-on-surface-lowest, var(--Gray-Modern-500, #667085))', @@ -241,6 +249,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} + {isPc ? ( {t('account_info:avatar')}  @@ -253,7 +262,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { border={theme.borders.base} overflow={'hidden'} boxShadow={'0 0 5px rgba(0,0,0,0.1)'} - onClick={onOpenSelectFile} + onClick={handleFileSelectorOpen} > @@ -264,7 +273,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { flexDirection={'column'} alignItems={'center'} cursor={'pointer'} - onClick={onOpenSelectFile} + onClick={handleFileSelectorOpen} > void }) => { )} + {feConfigs?.isPlus && ( {t('account_info:member_name')}  @@ -334,21 +344,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => { )} {isOpenUpdatePsw && } {isOpenUpdateContact && } - - onSelectImage(e, { - maxW: 300, - maxH: 300, - callback: (src) => { - if (!userInfo) return; - onclickSave({ - ...userInfo, - avatar: src - }); - } - }) - } - /> ); }; diff --git a/projects/app/src/pages/api/common/file/avatar.ts b/projects/app/src/pages/api/common/file/avatar.ts new file mode 100644 index 000000000000..ca5ab4ea242d --- /dev/null +++ b/projects/app/src/pages/api/common/file/avatar.ts @@ -0,0 +1,25 @@ +import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import { NextAPI } from '@/service/middleware/entry'; +import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/types'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; + +export type updateAvatarQuery = {}; + +export type updateAvatarBody = { + filename: string; + temporay: boolean; +}; + +export type updateAvatarResponse = CreatePostPresignedUrlResult; + +async function handler( + req: ApiRequestProps, + _: ApiResponseType +): Promise { + const { filename, temporay } = req.body; + const { teamId } = await authCert({ req, authToken: true }); + return await getS3AvatarSource().createPostPresignedUrl({ teamId, filename }, { temporay }); +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/app/update.ts b/projects/app/src/pages/api/core/app/update.ts index 5709f8ccc7f9..0ab2cd802092 100644 --- a/projects/app/src/pages/api/core/app/update.ts +++ b/projects/app/src/pages/api/core/app/update.ts @@ -27,6 +27,7 @@ import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { getI18nAppType } from '@fastgpt/service/support/user/audit/util'; import { i18nT } from '@fastgpt/web/i18n/utils'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; export type AppUpdateQuery = { appId: string; @@ -111,13 +112,19 @@ async function handler(req: ApiRequestProps) { nodes }); - await refreshSourceAvatar(avatar, app.avatar, session); + const permanentAvatar = await (async () => { + if (avatar) { + const s3AvatarSource = getS3AvatarSource(); + await s3AvatarSource.removeAvatar(app.avatar); // 移除旧的头像 + return await s3AvatarSource.moveAvatarFromTemp(avatar); // 永久化临时的预览头像 + } + })(); if (app.type === AppTypeEnum.toolSet && avatar) { await MongoApp.updateMany( { parentId: appId, teamId: app.teamId }, { - avatar + avatar: permanentAvatar }, { session } ); @@ -129,7 +136,7 @@ async function handler(req: ApiRequestProps) { ...parseParentIdInMongo(parentId), ...(name && { name }), ...(type && { type }), - ...(avatar && { avatar }), + ...(avatar && { avatar: permanentAvatar }), ...(intro !== undefined && { intro }), ...(teamTags && { teamTags }), ...(nodes && { diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts index a2b0611ad353..5cd9ad7d51e7 100644 --- a/projects/app/src/pages/api/core/dataset/update.ts +++ b/projects/app/src/pages/api/core/dataset/update.ts @@ -39,6 +39,7 @@ import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model'; import { computedCollectionChunkSettings } from '@fastgpt/global/core/dataset/training/utils'; import { getResourceOwnedClbs } from '@fastgpt/service/support/permission/controller'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar'; export type DatasetUpdateQuery = {}; export type DatasetUpdateResponse = any; @@ -202,12 +203,20 @@ async function handler( return flattenObjectWithConditions(apiDatasetServer); })(); + const permanentAvatar = await (async () => { + if (avatar) { + const s3AvatarSource = getS3AvatarSource(); + await s3AvatarSource.removeAvatar(dataset.avatar); // 移除旧的头像 + return await s3AvatarSource.moveAvatarFromTemp(avatar); // 永久化临时的预览头像 + } + })(); + await MongoDataset.findByIdAndUpdate( id, { ...parseParentIdInMongo(parentId), ...(name && { name }), - ...(avatar && { avatar }), + ...(avatar && { avatar: permanentAvatar }), ...(agentModel && { agentModel }), ...(vlmModel && { vlmModel }), ...(websiteConfig && { websiteConfig }), @@ -224,8 +233,6 @@ async function handler( dataset, autoSync }); - - await refreshSourceAvatar(avatar, dataset.avatar, session); }; await mongoSessionRun(async (session) => { diff --git a/projects/app/src/pages/api/support/user/account/update.ts b/projects/app/src/pages/api/support/user/account/update.ts index dfa451e29028..89da6a801f1e 100644 --- a/projects/app/src/pages/api/support/user/account/update.ts +++ b/projects/app/src/pages/api/support/user/account/update.ts @@ -8,6 +8,7 @@ import { NextAPI } from '@/service/middleware/entry'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller'; import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; export type UserAccountUpdateQuery = {}; export type UserAccountUpdateBody = UserUpdateParams; @@ -37,6 +38,10 @@ async function handler( } // if avatar, update team member avatar if (avatar) { + if (tmb?.avatar) { + await getS3AvatarSource().removeAvatar(tmb.avatar); + } + await MongoTeamMember.updateOne( { _id: tmbId @@ -45,7 +50,6 @@ async function handler( avatar } ).session(session); - await refreshSourceAvatar(avatar, tmb?.avatar, session); } }); diff --git a/projects/app/src/pages/api/system/img/[...id].ts b/projects/app/src/pages/api/system/img/[...id].ts new file mode 100644 index 000000000000..90d0df6be51d --- /dev/null +++ b/projects/app/src/pages/api/system/img/[...id].ts @@ -0,0 +1,32 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import path from 'path'; + +import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; +import { Types } from '@fastgpt/service/common/mongo'; +import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources'; + +// get the models available to the system +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { id } = req.query as { id: string[] }; + + const joined = id.join('/'); + const parsed = path.parse(joined); + const keys = path.format({ dir: parsed.dir, name: parsed.name, ext: '' }); + + if (Types.ObjectId.isValid(keys)) { + const { binary, mime } = await readMongoImg({ id: joined }); + res.setHeader('Content-Type', mime); + res.send(binary); + return; + } + + res.redirect(301, getS3AvatarSource().createPublicUrl(joined)); + } catch (error) { + jsonRes(res, { + code: 500, + error + }); + } +} diff --git a/projects/app/src/pages/api/system/img/[id].ts b/projects/app/src/pages/api/system/img/[id].ts deleted file mode 100644 index 882d21ab3c87..000000000000 --- a/projects/app/src/pages/api/system/img/[id].ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; - -import { readMongoImg } from '@fastgpt/service/common/file/image/controller'; - -// get the models available to the system -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { id } = req.query as { id: string }; - - const { binary, mime } = await readMongoImg({ id }); - - res.setHeader('Content-Type', mime); - res.send(binary); - } catch (error) { - jsonRes(res, { - code: 500, - error - }); - } -} diff --git a/projects/app/src/web/common/file/api.ts b/projects/app/src/web/common/file/api.ts index c73dad821095..60bed8f0014f 100644 --- a/projects/app/src/web/common/file/api.ts +++ b/projects/app/src/web/common/file/api.ts @@ -1,6 +1,6 @@ -import { DELETE, GET, POST } from '@/web/common/api/request'; +import { POST } from '@/web/common/api/request'; import type { UploadImgProps } from '@fastgpt/global/common/file/api.d'; -import type { UploadPresignedURLResponse } from '@fastgpt/service/common/s3/type'; +import type { CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/types'; import { type AxiosProgressEvent } from 'axios'; export const postUploadImg = (e: UploadImgProps) => POST('/common/file/uploadImage', e); @@ -32,3 +32,7 @@ export const postS3UploadFile = ( }, onUploadProgress }); + +export const getUploadAvatarPresignedUrl = (params: { filename: string; temporay: boolean }) => { + return POST('/common/file/avatar', params); +}; diff --git a/projects/app/src/web/common/file/utils.ts b/projects/app/src/web/common/file/utils.ts index 160ad29ea3ac..38358902d874 100644 --- a/projects/app/src/web/common/file/utils.ts +++ b/projects/app/src/web/common/file/utils.ts @@ -32,3 +32,15 @@ export const fileToBase64 = (file: File) => { reader.onerror = (error) => reject(error); }); }; + +export const base64ToFile = (base64: string, filename: string) => { + const arr = base64.split(','); + const mime = arr[0].match(/:(.*?);/)?.[1]; + const bstr = atob(arr[1]); + let n = bstr.length; + const u8arr = new Uint8Array(n); + while (n--) { + u8arr[n] = bstr.charCodeAt(n); + } + return new File([u8arr], filename, { type: mime }); +};