Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 5aaf1231 by heheer Committed by GitHub
Options

chat file url white list (#6053) 

* chat file url white list

* perf: white list

---------

Co-authored-by: archer <545436317@qq.com>
parent 2ccb5b50
Showing with 87 additions and 7 deletions
document/content/docs/upgrading/4-14/4144.mdx
...@@ -29,6 +29,7 @@ curl --location --request POST 'https://{{host}}/api/admin/initv4144' \ ...@@ -29,6 +29,7 @@ curl --location --request POST 'https://{{host}}/api/admin/initv4144' \
3. 对话日志支持展示 IP 地址归属地。 3. 对话日志支持展示 IP 地址归属地。
4. 通过 API 上传本地文件至知识库，保存至 S3。同时将旧版 Gridfs 代码全部移除。 4. 通过 API 上传本地文件至知识库，保存至 S3。同时将旧版 Gridfs 代码全部移除。
5. 新版订阅套餐逻辑。 5. 新版订阅套餐逻辑。
6. 支持配置对话文件白名单。
## ⚙️ 优化 ## ⚙️ 优化
...@@ -36,6 +37,7 @@ curl --location --request POST 'https://{{host}}/api/admin/initv4144' \ ...@@ -36,6 +37,7 @@ curl --location --request POST 'https://{{host}}/api/admin/initv4144' \
2. 问题优化采用 JinaAI 的边际收益公式，获取最大边际收益的检索词。 2. 问题优化采用 JinaAI 的边际收益公式，获取最大边际收益的检索词。
3. 用户通知，支持中英文，以及优化模板。 3. 用户通知，支持中英文，以及优化模板。
4. 删除知识库采用队列异步删除模式。 4. 删除知识库采用队列异步删除模式。
5. LLM 请求时，图片无效报错提示。
## 🐛 修复 ## 🐛 修复
......
document/data/doc-last-modified.json
...@@ -118,7 +118,7 @@ ...@@ -118,7 +118,7 @@
"document/content/docs/upgrading/4-14/4141.mdx": "2025-11-19T10:15:27+08:00", "document/content/docs/upgrading/4-14/4141.mdx": "2025-11-19T10:15:27+08:00",
"document/content/docs/upgrading/4-14/4142.mdx": "2025-11-18T19:27:14+08:00", "document/content/docs/upgrading/4-14/4142.mdx": "2025-11-18T19:27:14+08:00",
"document/content/docs/upgrading/4-14/4143.mdx": "2025-11-26T20:52:05+08:00", "document/content/docs/upgrading/4-14/4143.mdx": "2025-11-26T20:52:05+08:00",
"document/content/docs/upgrading/4-14/4144.mdx": "2025-12-07T14:24:15+08:00", "document/content/docs/upgrading/4-14/4144.mdx": "2025-12-08T01:44:15+08:00",
"document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00", "document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00", "document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00", "document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00",
......
packages/global/common/system/types/index.d.ts
...@@ -146,6 +146,7 @@ export type SystemEnvType = { ...@@ -146,6 +146,7 @@ export type SystemEnvType = {
chatApiKey?: string; chatApiKey?: string;
customPdfParse?: customPdfParseType; customPdfParse?: customPdfParseType;
fileUrlWhitelist?: string[];
}; };
export type customPdfParseType = { export type customPdfParseType = {
......
packages/service/common/security/fileUrlValidator.ts 0 → 100644
const systemWhiteList = (() => {
const list: string[] = [];
if (process.env.S3_ENDPOINT) {
list.push(process.env.S3_ENDPOINT);
}
if (process.env.S3_EXTERNAL_BASE_URL) {
try {
const urlData = new URL(process.env.S3_EXTERNAL_BASE_URL);
list.push(urlData.hostname);
} catch (error) {}
}
if (process.env.FE_DOMAIN) {
try {
const urlData = new URL(process.env.FE_DOMAIN);
list.push(urlData.hostname);
} catch (error) {}
}
if (process.env.PRO_URL) {
try {
const urlData = new URL(process.env.PRO_URL);
list.push(urlData.hostname);
} catch (error) {}
}
return list;
})();
export const validateFileUrlDomain = (url: string): boolean => {
try {
// Allow all URLs if the whitelist is empty
if ((global.systemEnv?.fileUrlWhitelist || []).length === 0) {
return true;
}
const whitelistArray = [...(global.systemEnv?.fileUrlWhitelist || []), ...systemWhiteList];
const urlObj = new URL(url);
const isAllowed = whitelistArray.some((domain) => {
if (!domain || typeof domain !== 'string') return false;
return urlObj.hostname === domain;
});
if (!isAllowed) {
return false;
}
return true;
} catch (error) {
return true;
}
};
packages/service/core/ai/llm/utils.ts
...@@ -14,6 +14,7 @@ import { addLog } from '../../../common/system/log'; ...@@ -14,6 +14,7 @@ import { addLog } from '../../../common/system/log';
import { getImageBase64 } from '../../../common/file/image/utils'; import { getImageBase64 } from '../../../common/file/image/utils';
import { getS3ChatSource } from '../../../common/s3/sources/chat'; import { getS3ChatSource } from '../../../common/s3/sources/chat';
import { isInternalAddress } from '../../../common/system/utils'; import { isInternalAddress } from '../../../common/system/utils';
import { getErrText } from '@fastgpt/global/common/error/utils';
export const filterGPTMessageByMaxContext = async ({ export const filterGPTMessageByMaxContext = async ({
messages = [], messages = [],
...@@ -166,6 +167,7 @@ export const loadRequestMessages = async ({ ...@@ -166,6 +167,7 @@ export const loadRequestMessages = async ({
process.env.MULTIPLE_DATA_TO_BASE64 !== 'false' || process.env.MULTIPLE_DATA_TO_BASE64 !== 'false' ||
isInternalAddress(imgUrl) isInternalAddress(imgUrl)
) { ) {
try {
const url = await (async () => { const url = await (async () => {
if (item.key) { if (item.key) {
try { try {
...@@ -186,6 +188,11 @@ export const loadRequestMessages = async ({ ...@@ -186,6 +188,11 @@ export const loadRequestMessages = async ({
url: base64 url: base64
} }
}; };
} catch (error) {
return Promise.reject(
`Cannot load image ${imgUrl}, because ${getErrText(error)}`
);
}
} }
// 检查下这个图片是否可以被访问，如果不行的话，则过滤掉 // 检查下这个图片是否可以被访问，如果不行的话，则过滤掉
......
packages/service/core/workflow/dispatch/index.ts
...@@ -23,7 +23,7 @@ import type { ...@@ -23,7 +23,7 @@ import type {
SystemVariablesType SystemVariablesType
} from '@fastgpt/global/core/workflow/runtime/type'; } from '@fastgpt/global/core/workflow/runtime/type';
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type.d'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type.d';
import { getErrText } from '@fastgpt/global/common/error/utils'; import { getErrText, UserError } from '@fastgpt/global/common/error/utils';
import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants'; import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';
import { filterPublicNodeResponseData } from '@fastgpt/global/core/chat/utils'; import { filterPublicNodeResponseData } from '@fastgpt/global/core/chat/utils';
import { import {
...@@ -58,6 +58,7 @@ import type { MCPClient } from '../../app/mcp'; ...@@ -58,6 +58,7 @@ import type { MCPClient } from '../../app/mcp';
import { TeamErrEnum } from '@fastgpt/global/common/error/code/team'; import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
import { i18nT } from '../../../../web/i18n/utils'; import { i18nT } from '../../../../web/i18n/utils';
import { clone } from 'lodash'; import { clone } from 'lodash';
import { validateFileUrlDomain } from '../../../common/security/fileUrlValidator';
type Props = Omit< type Props = Omit<
ChatDispatchProps, ChatDispatchProps,
...@@ -88,7 +89,21 @@ export async function dispatchWorkFlow({ ...@@ -88,7 +89,21 @@ export async function dispatchWorkFlow({
}: Props & WorkflowUsageProps): Promise<DispatchFlowResponse> { }: Props & WorkflowUsageProps): Promise<DispatchFlowResponse> {
const { res, stream, runningUserInfo, runningAppInfo, lastInteractive, histories, query } = data; const { res, stream, runningUserInfo, runningAppInfo, lastInteractive, histories, query } = data;
// Check url valid
const invalidInput = query.some((item) => {
if (item.type === ChatItemValueTypeEnum.file && item.file?.url) {
if (!validateFileUrlDomain(item.file.url)) {
return true;
}
}
});
if (invalidInput) {
addLog.info('[Workflow run] Invalid file url');
return Promise.reject(new UserError('Invalid file url'));
}
// Check point
await checkTeamAIPoints(runningUserInfo.teamId); await checkTeamAIPoints(runningUserInfo.teamId);
const [{ timezone, externalProvider }, newUsageId] = await Promise.all([ const [{ timezone, externalProvider }, newUsageId] = await Promise.all([
getUserChatInfo(runningUserInfo.tmbId), getUserChatInfo(runningUserInfo.tmbId),
(() => { (() => {
......
projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx
...@@ -527,7 +527,7 @@ const ChatBox = ({ ...@@ -527,7 +527,7 @@ const ChatBox = ({
file: { file: {
type: file.type, type: file.type,
name: file.name, name: file.name,
url: file.url || '', url: file.key ? undefined : file.url || '',
icon: file.icon || '', icon: file.icon || '',
key: file.key || '' key: file.key || ''
} }
......
projects/app/src/pages/api/v1/chat/completions.ts
...@@ -6,8 +6,10 @@ import { addLog } from '@fastgpt/service/common/system/log'; ...@@ -6,8 +6,10 @@ import { addLog } from '@fastgpt/service/common/system/log';
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants'; import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch'; import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d'; import type {
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d'; ChatCompletionCreateParams,
ChatCompletionMessageParam
} from '@fastgpt/global/core/ai/type.d';
import { import {
getWorkflowEntryNodeIds, getWorkflowEntryNodeIds,
getMaxHistoryLimitFromNodes, getMaxHistoryLimitFromNodes,
......
projects/app/src/pages/api/v2/chat/completions.ts
...@@ -6,8 +6,10 @@ import { addLog } from '@fastgpt/service/common/system/log'; ...@@ -6,8 +6,10 @@ import { addLog } from '@fastgpt/service/common/system/log';
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants'; import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch'; import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d'; import type {
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d'; ChatCompletionCreateParams,
ChatCompletionMessageParam
} from '@fastgpt/global/core/ai/type.d';
import { import {
getWorkflowEntryNodeIds, getWorkflowEntryNodeIds,
getMaxHistoryLimitFromNodes, getMaxHistoryLimitFromNodes,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment