Skip to content

Commit 5aaf123

Browse files
newfish-cmykc121914yu
newfish-cmyk
and
c121914yu
authored
chat file url white list (#6053)
* chat file url white list * perf: white list --------- Co-authored-by: archer <[email protected]>
1 parent 2ccb5b5 commit 5aaf123

File tree

9 files changed

+107
-27
lines changed

9 files changed

+107
-27
lines changed

‎document/content/docs/upgrading/4-14/4144.mdx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,15 @@ curl --location --request POST 'https://{{host}}/api/admin/initv4144' \
2929
3. 对话日志支持展示 IP 地址归属地。
3030
4. 通过 API 上传本地文件至知识库，保存至 S3。同时将旧版 Gridfs 代码全部移除。
3131
5. 新版订阅套餐逻辑。
32+
6. 支持配置对话文件白名单。
3233

3334
## ⚙️ 优化
3435

3536
1. 增加 S3 上传文件超时时长为 5 分钟。
3637
2. 问题优化采用 JinaAI 的边际收益公式，获取最大边际收益的检索词。
3738
3. 用户通知，支持中英文，以及优化模板。
3839
4. 删除知识库采用队列异步删除模式。
40+
5. LLM 请求时，图片无效报错提示。
3941

4042
## 🐛 修复
4143

‎document/data/doc-last-modified.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@
118118
"document/content/docs/upgrading/4-14/4141.mdx": "2025-11-19T10:15:27+08:00",
119119
"document/content/docs/upgrading/4-14/4142.mdx": "2025-11-18T19:27:14+08:00",
120120
"document/content/docs/upgrading/4-14/4143.mdx": "2025-11-26T20:52:05+08:00",
121-
"document/content/docs/upgrading/4-14/4144.mdx": "2025-12-07T14:24:15+08:00",
121+
"document/content/docs/upgrading/4-14/4144.mdx": "2025-12-08T01:44:15+08:00",
122122
"document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00",
123123
"document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00",
124124
"document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00",

‎packages/global/common/system/types/index.d.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ export type SystemEnvType = {
146146
chatApiKey?: string;
147147

148148
customPdfParse?: customPdfParseType;
149+
fileUrlWhitelist?: string[];
149150
};
150151

151152
export type customPdfParseType = {

‎packages/service/common/security/fileUrlValidator.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
const systemWhiteList = (() => {
2+
const list: string[] = [];
3+
if (process.env.S3_ENDPOINT) {
4+
list.push(process.env.S3_ENDPOINT);
5+
}
6+
if (process.env.S3_EXTERNAL_BASE_URL) {
7+
try {
8+
const urlData = new URL(process.env.S3_EXTERNAL_BASE_URL);
9+
list.push(urlData.hostname);
10+
} catch (error) {}
11+
}
12+
if (process.env.FE_DOMAIN) {
13+
try {
14+
const urlData = new URL(process.env.FE_DOMAIN);
15+
list.push(urlData.hostname);
16+
} catch (error) {}
17+
}
18+
if (process.env.PRO_URL) {
19+
try {
20+
const urlData = new URL(process.env.PRO_URL);
21+
list.push(urlData.hostname);
22+
} catch (error) {}
23+
}
24+
return list;
25+
})();
26+
27+
export const validateFileUrlDomain = (url: string): boolean => {
28+
try {
29+
// Allow all URLs if the whitelist is empty
30+
if ((global.systemEnv?.fileUrlWhitelist || []).length === 0) {
31+
return true;
32+
}
33+
34+
const whitelistArray = [...(global.systemEnv?.fileUrlWhitelist || []), ...systemWhiteList];
35+
36+
const urlObj = new URL(url);
37+
38+
const isAllowed = whitelistArray.some((domain) => {
39+
if (!domain || typeof domain !== 'string') return false;
40+
return urlObj.hostname === domain;
41+
});
42+
43+
if (!isAllowed) {
44+
return false;
45+
}
46+
47+
return true;
48+
} catch (error) {
49+
return true;
50+
}
51+
};

‎packages/service/core/ai/llm/utils.ts‎

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import { addLog } from '../../../common/system/log';
1414
import { getImageBase64 } from '../../../common/file/image/utils';
1515
import { getS3ChatSource } from '../../../common/s3/sources/chat';
1616
import { isInternalAddress } from '../../../common/system/utils';
17+
import { getErrText } from '@fastgpt/global/common/error/utils';
1718

1819
export const filterGPTMessageByMaxContext = async ({
1920
messages = [],
@@ -166,26 +167,32 @@ export const loadRequestMessages = async ({
166167
process.env.MULTIPLE_DATA_TO_BASE64 !== 'false' ||
167168
isInternalAddress(imgUrl)
168169
) {
169-
const url = await (async () => {
170-
if (item.key) {
171-
try {
172-
return await getS3ChatSource().createGetChatFileURL({
173-
key: item.key,
174-
external: false
175-
});
176-
} catch (error) {}
177-
}
178-
return imgUrl;
179-
})();
180-
const { completeBase64: base64 } = await getImageBase64(url);
181-
182-
return {
183-
...item,
184-
image_url: {
185-
...item.image_url,
186-
url: base64
187-
}
188-
};
170+
try {
171+
const url = await (async () => {
172+
if (item.key) {
173+
try {
174+
return await getS3ChatSource().createGetChatFileURL({
175+
key: item.key,
176+
external: false
177+
});
178+
} catch (error) {}
179+
}
180+
return imgUrl;
181+
})();
182+
const { completeBase64: base64 } = await getImageBase64(url);
183+
184+
return {
185+
...item,
186+
image_url: {
187+
...item.image_url,
188+
url: base64
189+
}
190+
};
191+
} catch (error) {
192+
return Promise.reject(
193+
`Cannot load image ${imgUrl}, because ${getErrText(error)}`
194+
);
195+
}
189196
}
190197

191198
// 检查下这个图片是否可以被访问，如果不行的话，则过滤掉

‎packages/service/core/workflow/dispatch/index.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import type {
2323
SystemVariablesType
2424
} from '@fastgpt/global/core/workflow/runtime/type';
2525
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type.d';
26-
import { getErrText } from '@fastgpt/global/common/error/utils';
26+
import { getErrText, UserError } from '@fastgpt/global/common/error/utils';
2727
import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';
2828
import { filterPublicNodeResponseData } from '@fastgpt/global/core/chat/utils';
2929
import {
@@ -58,6 +58,7 @@ import type { MCPClient } from '../../app/mcp';
5858
import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
5959
import { i18nT } from '../../../../web/i18n/utils';
6060
import { clone } from 'lodash';
61+
import { validateFileUrlDomain } from '../../../common/security/fileUrlValidator';
6162

6263
type Props = Omit<
6364
ChatDispatchProps,
@@ -88,7 +89,21 @@ export async function dispatchWorkFlow({
8889
}: Props & WorkflowUsageProps): Promise<DispatchFlowResponse> {
8990
const { res, stream, runningUserInfo, runningAppInfo, lastInteractive, histories, query } = data;
9091

92+
// Check url valid
93+
const invalidInput = query.some((item) => {
94+
if (item.type === ChatItemValueTypeEnum.file && item.file?.url) {
95+
if (!validateFileUrlDomain(item.file.url)) {
96+
return true;
97+
}
98+
}
99+
});
100+
if (invalidInput) {
101+
addLog.info('[Workflow run] Invalid file url');
102+
return Promise.reject(new UserError('Invalid file url'));
103+
}
104+
// Check point
91105
await checkTeamAIPoints(runningUserInfo.teamId);
106+
92107
const [{ timezone, externalProvider }, newUsageId] = await Promise.all([
93108
getUserChatInfo(runningUserInfo.tmbId),
94109
(() => {

‎projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -527,7 +527,7 @@ const ChatBox = ({
527527
file: {
528528
type: file.type,
529529
name: file.name,
530-
url: file.url || '',
530+
url: file.key ? undefined : file.url || '',
531531
icon: file.icon || '',
532532
key: file.key || ''
533533
}

‎projects/app/src/pages/api/v1/chat/completions.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ import { addLog } from '@fastgpt/service/common/system/log';
66
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
77
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
88
import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
9-
import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d';
10-
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d';
9+
import type {
10+
ChatCompletionCreateParams,
11+
ChatCompletionMessageParam
12+
} from '@fastgpt/global/core/ai/type.d';
1113
import {
1214
getWorkflowEntryNodeIds,
1315
getMaxHistoryLimitFromNodes,

‎projects/app/src/pages/api/v2/chat/completions.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ import { addLog } from '@fastgpt/service/common/system/log';
66
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
77
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
88
import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
9-
import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d';
10-
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d';
9+
import type {
10+
ChatCompletionCreateParams,
11+
ChatCompletionMessageParam
12+
} from '@fastgpt/global/core/ai/type.d';
1113
import {
1214
getWorkflowEntryNodeIds,
1315
getMaxHistoryLimitFromNodes,

0 commit comments

Comments
 (0)