feat(s3): add proxy transfer mode with tokenized upload/download

Author: xqvvu

document/content/docs/self-host/config/object-storage.en.mdx

@@ -21,14 +21,17 @@ This guide covers environment variable configuration for object storage provider
 - `STORAGE_SECRET_ACCESS_KEY` Secret Access Key for the service credentials
 - `STORAGE_PUBLIC_BUCKET` FastGPT public resource bucket name
 - `STORAGE_PRIVATE_BUCKET` FastGPT private resource bucket name
+- `STORAGE_TRANSFER_MODE` File transfer mode. Options:
+  - `proxy` (default): uploads and downloads are proxied by FastGPT backend; browser direct access to object storage is not required.
+  - `presigned`: uploads and downloads use object-storage presigned URLs (legacy-compatible mode).
 
 ### Self-Hosted MinIO and AWS S3
 
 > MinIO has strong AWS S3 protocol support, so MinIO and AWS S3 configurations are nearly identical — differences come from provider-specific or self-hosted requirements.
 > In theory, any object storage with S3 protocol support comparable to MinIO will work, such as SeaweedFS, RustFS, etc.
 
 - `STORAGE_S3_ENDPOINT` Internal connection address. Can be a container ID, e.g., `http://fastgpt-minio:9000`
-- `STORAGE_EXTERNAL_ENDPOINT` An address accessible by both **server** and **client** to reach the bucket. Use a fixed host IP or domain name — don't use `127.0.0.1` or `localhost` (containers can't access loopback addresses). This address is used when generating signed file upload URLs.
+- `STORAGE_EXTERNAL_ENDPOINT` An address accessible by both **server** and **client** to reach the bucket. Use a fixed host IP or domain name — don't use `127.0.0.1` or `localhost` (containers can't access loopback addresses). When `STORAGE_TRANSFER_MODE=presigned`, this is used to generate upload/download presigned URLs.
 - `STORAGE_S3_FORCE_PATH_STYLE` [Optional] Virtual-hosted-style or path-style routing. If vendor is `minio`, this is fixed to `true`.
 - `STORAGE_S3_MAX_RETRIES` [Optional] Maximum request retry attempts. Default: 3
 
@@ -43,6 +46,7 @@ STORAGE_ACCESS_KEY_ID=your_access_key
 STORAGE_SECRET_ACCESS_KEY=your_secret_key
 STORAGE_PUBLIC_BUCKET=fastgpt-public
 STORAGE_PRIVATE_BUCKET=fastgpt-private
+STORAGE_TRANSFER_MODE=proxy
 STORAGE_EXTERNAL_ENDPOINT=http://127.0.0.1:9000
 STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
 STORAGE_S3_FORCE_PATH_STYLE=true

document/content/docs/self-host/config/object-storage.mdx

@@ -21,14 +21,17 @@ import FastGPTLink from '@/components/docs/linkFastGPT';
 - `STORAGE_SECRET_ACCESS_KEY` 服务访问密钥的 Secret Access Key
 - `STORAGE_PUBLIC_BUCKET` FastGPT 公开资源存储桶桶名
 - `STORAGE_PRIVATE_BUCKET` FastGPT 私有资源存储桶桶名
+- `STORAGE_TRANSFER_MODE` 文件传输模式，可选值：
+  - `proxy`（默认）：上传和下载都通过 FastGPT 后端代理，不要求浏览器直连对象存储。
+  - `presigned`：上传和下载都通过对象存储预签名 URL，兼容旧模式。
 
 ### 自部署的 MinIO 和 AWS S3
 
 > MinIO 这类产品对 AWS S3 协议支持比较完整，因此使用 Minio 和AWS S3 配置几乎可以是相同的，只是因为服务商提供和自部署的区别，会有额外的配置。
 > 因此理论上任何对 AWS S3 协议的支持程度至少和 MinIO 相当的对象存储服务都可以使用，比如 SeaweedFS、RustFS 等。
 
 - `STORAGE_S3_ENDPOINT` 内网连接地址，可以是容器 id 连接，比如 `http://fastgpt-minio:9000`
-- `STORAGE_EXTERNAL_ENDPOINT` 一个**服务器**和**客户端**均可访问到存储桶的地址，可以是固定的宿主机 IP 或者域名，注意不要填写成 127.0.0.1 或者 localhost 等本地回环地址(因为容器里无法使用)。该地址用于签发文件上传 URL 时使用。
+- `STORAGE_EXTERNAL_ENDPOINT` 一个**服务器**和**客户端**均可访问到存储桶的地址，可以是固定的宿主机 IP 或者域名，注意不要填写成 127.0.0.1 或者 localhost 等本地回环地址(因为容器里无法使用)。当 `STORAGE_TRANSFER_MODE=presigned` 时，该配置用于签发上传/下载预签名 URL。
 - `STORAGE_S3_FORCE_PATH_STYLE` 【可选】虚拟主机风格路由或路径路由风格，其中如果厂商填写了 `minio` 的话，该值被固定为 `true`
 - `STORAGE_S3_MAX_RETRIES` 【可选】请求最大尝试次数，默认为 3 次
 
@@ -43,6 +46,7 @@ STORAGE_ACCESS_KEY_ID=your_access_key
 STORAGE_SECRET_ACCESS_KEY=your_secret_key
 STORAGE_PUBLIC_BUCKET=fastgpt-public
 STORAGE_PRIVATE_BUCKET=fastgpt-private
+STORAGE_TRANSFER_MODE=proxy
 STORAGE_EXTERNAL_ENDPOINT=http://127.0.0.1:9000
 STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
 STORAGE_S3_FORCE_PATH_STYLE=true

packages/global/common/error/s3.ts

@@ -25,6 +25,22 @@ export function parseS3UploadError({
   if (error?.response?.data) {
     const data = error.response.data;
 
+    if (typeof data === 'object' && data !== null) {
+      const msg = `${data.message || ''}`.trim();
+      const statusText = `${data.statusText || ''}`.trim();
+
+      if (msg.includes('EntityTooLarge') || statusText.includes('EntityTooLarge')) {
+        return t('common:error:s3_upload_file_too_large', { max: maxSizeStr });
+      }
+      if (
+        msg.includes('unAuthFile') ||
+        statusText.includes('unAuthFile') ||
+        msg.includes('unAuthorization')
+      ) {
+        return t('common:error:s3_upload_auth_failed');
+      }
+    }
+
     // Try to parse XML error response
     if (typeof data === 'string') {
       if (data.includes('EntityTooLarge')) {

packages/global/common/string/tools.ts

@@ -188,6 +188,23 @@ export const sliceStrStartEnd = (str: string | null = '', start: number, end: nu
     => pdf
 */
 export const parseFileExtensionFromUrl = (url = '') => {
+  // Prefer explicit filename in query params for proxy links:
+  // e.g. /api/system/file/download/<token>?filename=image.jpg
+  try {
+    const parsedUrl = new URL(url, 'http://localhost');
+    const queryFilename =
+      parsedUrl.searchParams.get('filename') || parsedUrl.searchParams.get('name');
+    if (queryFilename) {
+      const extFromQuery = path.extname(decodeURIComponent(queryFilename));
+      if (extFromQuery.startsWith('.')) {
+        return extFromQuery.slice(1).toLowerCase();
+      }
+    }
+  } catch {
+    // noop
+    // fallback to legacy parser below
+  }
+
   // Remove query params and hash first
   const urlWithoutQuery = url.split('?')[0].split('#')[0];
   const extension = path.extname(urlWithoutQuery);

packages/service/common/file/read/utils.ts

@@ -228,7 +228,6 @@ export const readFileContentByBuffer = async ({
         }
       })();
       rawText = rawText.replace(item.uuid, src);
-      // rawText = rawText.replace(item.uuid, jwtSignS3ObjectKey(src, addDays(new Date(), 90)));
       if (formatText) {
         formatText = formatText.replace(item.uuid, src);
       }

packages/service/common/s3/buckets/base.ts

@@ -13,7 +13,7 @@ import {
   type createPreviewUrlParams,
   CreateGetPresignedUrlParamsSchema
 } from '../type';
-import { getSystemMaxFileSize, Mimes } from '../constants';
+import { storageTransferMode, getSystemMaxFileSize, Mimes } from '../constants';
 import path from 'node:path';
 import { MongoS3TTL } from '../schema';
 import { addHours, addMinutes, differenceInSeconds } from 'date-fns';
@@ -22,6 +22,7 @@ import { addS3DelJob } from '../mq';
 import { type UploadFileByBufferParams, UploadFileByBufferSchema } from '../type';
 import type { createStorage } from '@fastgpt-sdk/storage';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
+import { jwtSignS3DownloadToken, jwtSignS3UploadToken } from '../token';
 
 const logger = getLogger(LogCategories.INFRA.S3);
 
@@ -139,18 +140,12 @@ export class S3BaseBucket {
       const ext = path.extname(filename).toLowerCase();
       const contentType = Mimes[ext as keyof typeof Mimes] ?? 'application/octet-stream';
       const expiredSeconds = differenceInSeconds(addMinutes(new Date(), 10), new Date());
-
-      const { metadata, url } = await this.externalClient.generatePresignedPutUrl({
-        key: params.rawKey,
-        expiredSeconds,
-        contentType,
-        metadata: {
-          contentDisposition: `attachment; filename="${encodeURIComponent(filename)}"`,
-          originFilename: encodeURIComponent(filename),
-          uploadTime: new Date().toISOString(),
-          ...params.metadata
-        }
-      });
+      const metadata = {
+        contentDisposition: `attachment; filename="${encodeURIComponent(filename)}"`,
+        originFilename: encodeURIComponent(filename),
+        uploadTime: new Date().toISOString(),
+        ...params.metadata
+      };
 
       if (expiredHours) {
         await MongoS3TTL.create({
@@ -160,11 +155,36 @@ export class S3BaseBucket {
         });
       }
 
+      if (storageTransferMode === 'proxy') {
+        return {
+          url: jwtSignS3UploadToken({
+            objectKey: params.rawKey,
+            bucketName: this.bucketName,
+            expiredTime: addMinutes(new Date(), Math.ceil(expiredSeconds / 60)),
+            maxSize: formatMaxFileSize,
+            contentType,
+            metadata
+          }),
+          key: params.rawKey,
+          headers: {
+            'content-type': contentType
+          },
+          maxSize: formatMaxFileSize
+        };
+      }
+
+      const { metadata: uploadHeaders, url } = await this.externalClient.generatePresignedPutUrl({
+        key: params.rawKey,
+        expiredSeconds,
+        contentType,
+        metadata
+      });
+
       return {
         url: url,
         key: params.rawKey,
         headers: {
-          ...metadata
+          ...uploadHeaders
         },
         maxSize: formatMaxFileSize
       };
@@ -184,6 +204,19 @@ export class S3BaseBucket {
     const { key, expiredHours } = parsed;
     const expires = expiredHours ? expiredHours * 60 * 60 : 30 * 60; // expires 的单位是秒 默认 30 分钟
 
+    if (storageTransferMode === 'proxy') {
+      return {
+        bucket: this.bucketName,
+        key,
+        url: jwtSignS3DownloadToken({
+          objectKey: key,
+          bucketName: this.bucketName,
+          expiredTime: addMinutes(new Date(), Math.ceil(expires / 60)),
+          filename: path.basename(key)
+        })
+      };
+    }
+
     return await this.externalClient.generatePresignedGetUrl({ key, expiredSeconds: expires });
   }
 

packages/service/common/s3/constants.ts

@@ -4,6 +4,7 @@ import type {
   IOssStorageOptions,
   IStorageOptions
 } from '@fastgpt-sdk/storage';
+import { env } from '../../env';
 
 export const Mimes = {
   '.gif': 'image/gif',
@@ -33,6 +34,8 @@ export const S3Buckets = {
   private: process.env.STORAGE_PRIVATE_BUCKET || 'fastgpt-private'
 } as const;
 
+export const storageTransferMode = env.STORAGE_TRANSFER_MODE;
+
 export const getSystemMaxFileSize = () => {
   const config = global.feConfigs.uploadFileMaxSize || 1024; // MB, default 1024MB
   return config; // bytes