Commit 62ed4655 by Finley Ge Committed by GitHub

fix(agent): pass uploaded file urls to tools (#7091)

* fix(agent): pass uploaded file urls to tools

* perf: inject prompt

* test: update agent file reminder assertions

* test: update read files output assertions

---------

Co-authored-by: archer <545436317@qq.com>
parent 870ea89b
...@@ -234,3 +234,35 @@ pnpm --filter @fastgpt/service test test/core/workflow/dispatch/ai/agent/adapter ...@@ -234,3 +234,35 @@ pnpm --filter @fastgpt/service test test/core/workflow/dispatch/ai/agent/adapter
- [x] HelperBot / ChatAgent UI 改为暴露 `read_files` - [x] HelperBot / ChatAgent UI 改为暴露 `read_files`
- [x] 补充核心单测。 - [x] 补充核心单测。
- [ ] 浏览器集成测试:上传文件 + 选择知识库 + 当前时间 + `read_files` 工具调用。 - [ ] 浏览器集成测试:上传文件 + 选择知识库 + 当前时间 + `read_files` 工具调用。
## 2026-06-10 补充:文件 URL 与工具调用参数
### 问题
Agent 和 AgentV2 的文件上下文只把上传文件暴露为内部 `id`。模型可以用这个 `id`
调用内置 `read_files`,但当用户选择的外部工具需要文件链接时,模型容易把 `id`
填进工具参数,工具无法访问真实文件。
### 方案
- `AgentInputFile` 继续保留稳定 `id`,同时在文件 reminder 中暴露 `type``url`
- `fileUrlMap` 登记所有可用上传文件,覆盖 document/image/audio/video。
- `filesMap` 继续只登记 document 文件,专供 `read_files` 使用。
- 用户工具执行前调用 `replaceAgentFileIdsWithUrls(...)`,只把完整命中的字符串、数组项、
对象字段值从文件 `id` 替换为 `url`
- 不做长文本 substring 替换,避免普通业务文本里出现同名字符串时被误改。
### 边界
- 内置 `read_files` 仍使用文件 `id`,不走 URL 替换。
- 当前 `url` 来自聊天上传时保存的 `previewUrl` 或外部变量传入的链接;本次不额外刷新
已过期的 S3 signed URL。
- 后续如果要彻底解决历史长会话中的过期链接,应把 `AgentInputFile` 扩展为保留 `key`
在构建本轮 reminder 时用 `key` 重新签发新的 access URL。
### 新增测试
- 文件 reminder 包含 `<id>``<name>``<type>``<url>`
- `fileUrlMap` 覆盖 document/image/audio/video,`filesMap` 只覆盖 document。
- Unified Agent 和 PiAgent prompt 都包含文件 URL。
- `replaceAgentFileIdsWithUrls(...)` 只替换完整命中的 id,不替换长文本里的局部命中。
...@@ -6,6 +6,7 @@ import { ...@@ -6,6 +6,7 @@ import {
sandboxToolMap sandboxToolMap
} from '../../../ai/sandbox/tools'; } from '../../../ai/sandbox/tools';
import { parseI18nString } from '../../../../common/i18n/utils'; import { parseI18nString } from '../../../../common/i18n/utils';
import { documentFileType } from '../../../../common/file/constants';
export enum SubAppIds { export enum SubAppIds {
ask = 'ask_agent', ask = 'ask_agent',
...@@ -25,7 +26,7 @@ export const systemSubInfo: Record< ...@@ -25,7 +26,7 @@ export const systemSubInfo: Record<
en: 'FileParsing' en: 'FileParsing'
}, },
avatar: 'core/workflow/template/readFiles', avatar: 'core/workflow/template/readFiles',
toolDescription: '读取文件内容,并返回文件内容。' toolDescription: `读取文档并返回文档内容,支持: ${documentFileType}`
}, },
[SubAppIds.datasetSearch]: { [SubAppIds.datasetSearch]: {
name: { name: {
......
import { ReadFileTooData } from '../../../workflow/dispatch/ai/toolcall/tools/file';
/** /**
* 将本轮上传文件整理为文本上下文,包含文件名、沙盒路径和可选文件内容。 * 将本轮上传文件整理为文本上下文,包含文件名、沙盒路径和可选文件内容。
*/ */
export const getUserFilesPrompt = ( export const getUserFilesPrompt = (
files: { id?: string; name: string; sandboxPath?: string; content?: string }[] = [] files: { id?: string; name: string; url: string; sandboxPath?: string; content?: string }[] = []
) => { ) => {
if (files.length === 0) return ''; if (files.length === 0) return '';
return `# Input Files return `## 对话文件
用户本次上传的文件: 用户本次对话上传的文件,用途:
1. 可通过 ${ReadFileTooData.id} 读取文档内容。
2. 可把 url 作为模型参数。
${files ${files
.map((file) => .map((file) =>
`<file> `<file>
${file.id ? `<id>${file.id}</id>` : ''} ${file.id ? `<id>${file.id}</id>` : ''}
<name>${file.name}</name> <name>${file.name}</name>
<url>${file.url}</url>
${file.sandboxPath ? `<sandboxPath>${file.sandboxPath}</sandboxPath>` : ''} ${file.sandboxPath ? `<sandboxPath>${file.sandboxPath}</sandboxPath>` : ''}
${file.content ? `<content>${file.content}</content>` : ''} ${file.content ? `<content>${file.content}</content>` : ''}
</file>`.trim() </file>`.trim()
......
...@@ -53,7 +53,7 @@ type AgentSelectedDatasetContext = AgentSelectedDatasetInput & { ...@@ -53,7 +53,7 @@ type AgentSelectedDatasetContext = AgentSelectedDatasetInput & {
export const isValidAgentFileUrl = (url: unknown): url is string => { export const isValidAgentFileUrl = (url: unknown): url is string => {
if (typeof url !== 'string') return false; if (typeof url !== 'string') return false;
const validPrefixList = ['/', 'http', 'ws', 'data:']; const validPrefixList = ['/', 'http', 'ws'];
return validPrefixList.some((prefix) => url.startsWith(prefix)); return validPrefixList.some((prefix) => url.startsWith(prefix));
}; };
...@@ -127,9 +127,6 @@ export function parseAgentInputFiles({ ...@@ -127,9 +127,6 @@ export function parseAgentInputFiles({
.filter(Boolean) as AgentInputFile[]; .filter(Boolean) as AgentInputFile[];
} }
const filterAgentDocumentFiles = (files: AgentInputFile[]) =>
files.filter((file) => file.type === ChatFileTypeEnum.file);
/** /**
* 解析本轮用户输入文件。 * 解析本轮用户输入文件。
* *
...@@ -151,16 +148,14 @@ export function buildCurrentAgentInputFiles({ ...@@ -151,16 +148,14 @@ export function buildCurrentAgentInputFiles({
); );
const currentQueryFilesByUrl = new Map(queryFiles.map((file) => [file.url, file])); const currentQueryFilesByUrl = new Map(queryFiles.map((file) => [file.url, file]));
return filterAgentDocumentFiles( return parseAgentInputFiles({
parseAgentInputFiles({ files: currentInputFiles.map(
files: currentInputFiles.map( (url) => currentQueryFilesByUrl.get(url) || { type: ChatFileTypeEnum.file, url }
(url) => currentQueryFilesByUrl.get(url) || { type: ChatFileTypeEnum.file, url } ),
), prefixId: currentDataId || getNanoid(),
prefixId: currentDataId || getNanoid(), requestOrigin,
requestOrigin, maxFiles
maxFiles });
})
);
} }
/** /**
...@@ -213,17 +208,20 @@ export const loadAgentDatasetContext = async ( ...@@ -213,17 +208,20 @@ export const loadAgentDatasetContext = async (
/* Prompt */ /* Prompt */
export const buildAgentInputFilesPrompt = (files: AgentInputFile[] = []) => { export const buildAgentInputFilesPrompt = (files: AgentInputFile[] = []) => {
const documentFiles = filterAgentDocumentFiles(files); if (files.length === 0) return '';
if (documentFiles.length === 0) return '';
return `## 文件 return `## 对话文件
用户本次对话上传的的文件, 可通过 ${SubAppIds.readFiles} 读取文件内容: 用户本次对话上传的文件,用途:
1. 可通过 ${SubAppIds.readFiles} 读取文档内容。
2. 可把 url 作为模型参数。
${documentFiles ${files
.map( .map(
(file) => `<file> (file) => `<file>
<id>${escapeXml(file.id)}</id> <id>${escapeXml(file.id)}</id>
<name>${escapeXml(file.name)}</name> <name>${escapeXml(file.name)}</name>
<type>${escapeXml(file.type)}</type>
<url>${escapeXml(file.url)}</url>
</file>` </file>`
) )
.join('\n')}`; .join('\n')}`;
...@@ -319,6 +317,7 @@ export type UseUserContextResult = { ...@@ -319,6 +317,7 @@ export type UseUserContextResult = {
chatHistories: ChatItemMiniType[]; chatHistories: ChatItemMiniType[];
currentFiles: AgentInputFile[]; currentFiles: AgentInputFile[];
queryInput: string; queryInput: string;
fileUrlMap: Record<string, string>;
filesMap: Record<string, string>; filesMap: Record<string, string>;
getCurrentMessages: (params?: { getCurrentMessages: (params?: {
skillInfos?: DeployedSkillInfo[]; skillInfos?: DeployedSkillInfo[];
...@@ -362,6 +361,8 @@ export const useUserContext = async ({ ...@@ -362,6 +361,8 @@ export const useUserContext = async ({
timezone: string; timezone: string;
}): Promise<UseUserContextResult> => { }): Promise<UseUserContextResult> => {
const chatHistories = getHistories(history, histories); const chatHistories = getHistories(history, histories);
// fileUrlMap 记录所有上传文件,供普通工具参数把 file id 兜底转换成可访问 URL。
const fileUrlMap: Record<string, string> = {};
// filesMap 只给 read_files 使用,因此只登记 document 类型文件。 // filesMap 只给 read_files 使用,因此只登记 document 类型文件。
const filesMap: Record<string, string> = {}; const filesMap: Record<string, string> = {};
...@@ -372,6 +373,7 @@ export const useUserContext = async ({ ...@@ -372,6 +373,7 @@ export const useUserContext = async ({
if (files.length === 0) return; if (files.length === 0) return;
for (const file of files) { for (const file of files) {
fileUrlMap[file.id] = file.url;
if (file.type === ChatFileTypeEnum.file) { if (file.type === ChatFileTypeEnum.file) {
filesMap[file.id] = file.url; filesMap[file.id] = file.url;
} }
...@@ -384,14 +386,12 @@ export const useUserContext = async ({ ...@@ -384,14 +386,12 @@ export const useUserContext = async ({
const { files } = chatValue2RuntimePrompt(message.value); const { files } = chatValue2RuntimePrompt(message.value);
const formatFiles = filterAgentDocumentFiles( const formatFiles = parseAgentInputFiles({
parseAgentInputFiles({ files,
files, prefixId: getMessagePrefixId(message, index),
prefixId: getMessagePrefixId(message, index), requestOrigin,
requestOrigin, maxFiles
maxFiles });
})
);
registerFiles(formatFiles); registerFiles(formatFiles);
if (formatFiles.length === 0) return message; if (formatFiles.length === 0) return message;
...@@ -439,6 +439,7 @@ export const useUserContext = async ({ ...@@ -439,6 +439,7 @@ export const useUserContext = async ({
chatHistories, chatHistories,
currentFiles: currentInputFiles, currentFiles: currentInputFiles,
queryInput, queryInput,
fileUrlMap,
filesMap, filesMap,
getCurrentMessages: ({ skillInfos, currentWorkingDirectory } = {}) => { getCurrentMessages: ({ skillInfos, currentWorkingDirectory } = {}) => {
const currentUserMessage: ChatItemMiniType = { const currentUserMessage: ChatItemMiniType = {
......
...@@ -186,7 +186,7 @@ export const dispatchRunAgent = async (props: DispatchAgentModuleProps): Promise ...@@ -186,7 +186,7 @@ export const dispatchRunAgent = async (props: DispatchAgentModuleProps): Promise
currentFiles: userContext.currentFiles currentFiles: userContext.currentFiles
}); });
// 获取请求上下文 // 获取请求上下文
const { chatHistories, queryInput, filesMap } = userContext; const { chatHistories, queryInput, filesMap, fileUrlMap } = userContext;
const { rewrittenHistories, currentUserMessage } = userContext.getCurrentMessages({ const { rewrittenHistories, currentUserMessage } = userContext.getCurrentMessages({
skillInfos, skillInfos,
currentWorkingDirectory currentWorkingDirectory
...@@ -267,6 +267,7 @@ export const dispatchRunAgent = async (props: DispatchAgentModuleProps): Promise ...@@ -267,6 +267,7 @@ export const dispatchRunAgent = async (props: DispatchAgentModuleProps): Promise
getSubAppInfo, getSubAppInfo,
getSubApp, getSubApp,
completionTools: agentCompletionTools, completionTools: agentCompletionTools,
fileUrlMap,
filesMap, filesMap,
sandboxClient, sandboxClient,
streamResponseFn: workflowStreamResponse streamResponseFn: workflowStreamResponse
......
...@@ -137,7 +137,7 @@ export const dispatchPiAgent = async (props: DispatchAgentModuleProps): Promise< ...@@ -137,7 +137,7 @@ export const dispatchPiAgent = async (props: DispatchAgentModuleProps): Promise<
currentFiles: userContext.currentFiles currentFiles: userContext.currentFiles
}); });
const { chatHistories, filesMap } = userContext; const { chatHistories, filesMap, fileUrlMap } = userContext;
const { currentUserMessage } = userContext.getCurrentMessages({ const { currentUserMessage } = userContext.getCurrentMessages({
skillInfos, skillInfos,
currentWorkingDirectory currentWorkingDirectory
...@@ -219,6 +219,7 @@ export const dispatchPiAgent = async (props: DispatchAgentModuleProps): Promise< ...@@ -219,6 +219,7 @@ export const dispatchPiAgent = async (props: DispatchAgentModuleProps): Promise<
getSubApp, getSubApp,
completionTools: agentCompletionTools, completionTools: agentCompletionTools,
sandboxClient, sandboxClient,
fileUrlMap,
filesMap filesMap
}; };
......
...@@ -125,18 +125,51 @@ export type ToolDispatchContext = Pick< ...@@ -125,18 +125,51 @@ export type ToolDispatchContext = Pick<
getSubAppInfo: GetSubAppInfoFnType; getSubAppInfo: GetSubAppInfoFnType;
getSubApp: (id: string) => SubAppRuntimeType | undefined; getSubApp: (id: string) => SubAppRuntimeType | undefined;
completionTools: ChatCompletionTool[]; completionTools: ChatCompletionTool[];
fileUrlMap?: Record<string, string>;
filesMap: Record<string, string>; filesMap: Record<string, string>;
sandboxClient?: SandboxClient; sandboxClient?: SandboxClient;
streamResponseFn?: (args: WorkflowResponseItemType) => void | undefined; streamResponseFn?: (args: WorkflowResponseItemType) => void | undefined;
}; };
/** /**
* 将工具参数中完整匹配的 Agent 文件 id 替换成真实 URL。
*
* LLM 有时会把文件清单里的 `<id>` 填到用户工具参数中;这些 id 只对 FastGPT
* 内置 read_files 有意义,普通工具更需要可访问链接。这里只替换完整字符串,
* 不处理长文本里的局部命中,避免误改业务字段。
*/
export const replaceAgentFileIdsWithUrls = <T>(value: T, fileUrlMap: Record<string, string>): T => {
if (!value || Object.keys(fileUrlMap).length === 0) return value;
const replaceValue = (input: unknown): unknown => {
if (typeof input === 'string') {
return fileUrlMap[input] || input;
}
if (Array.isArray(input)) {
return input.map((item) => replaceValue(item));
}
if (input && typeof input === 'object') {
return Object.fromEntries(
Object.entries(input).map(([key, item]) => [key, replaceValue(item)])
);
}
return input;
};
return replaceValue(value) as T;
};
/**
* 创建 workflow 工具执行器。 * 创建 workflow 工具执行器。
* 该执行器屏蔽工具来源差异,将沙盒、文件读取、知识库搜索和用户子应用统一成 agentLoop 可消费的工具结果。 * 该执行器屏蔽工具来源差异,将沙盒、文件读取、知识库搜索和用户子应用统一成 agentLoop 可消费的工具结果。
*/ */
export const getExecuteTool = ({ export const getExecuteTool = ({
getSubAppInfo, getSubAppInfo,
getSubApp, getSubApp,
fileUrlMap = {},
filesMap, filesMap,
sandboxClient, sandboxClient,
checkIsStopping, checkIsStopping,
...@@ -262,10 +295,13 @@ export const getExecuteTool = ({ ...@@ -262,10 +295,13 @@ export const getExecuteTool = ({
response: 'Params is not object' response: 'Params is not object'
}; };
} }
const requestParams = { const requestParams = replaceAgentFileIdsWithUrls(
...tool.params, {
...toolCallParams ...tool.params,
}; ...toolCallParams
},
fileUrlMap
);
if (tool.type === 'tool') { if (tool.type === 'tool') {
const { response, usages, nodeResponse } = await dispatchTool({ const { response, usages, nodeResponse } = await dispatchTool({
......
...@@ -447,6 +447,7 @@ const getChatMessages = async ({ ...@@ -447,6 +447,7 @@ const getChatMessages = async ({
return files.map((file) => ({ return files.map((file) => ({
name: file.name, name: file.name,
url: file.url,
content: file.content content: file.content
})); }));
} }
......
...@@ -10,6 +10,7 @@ import { sliceStrStartEnd } from '@fastgpt/global/common/string/tools'; ...@@ -10,6 +10,7 @@ import { sliceStrStartEnd } from '@fastgpt/global/common/string/tools';
import z from 'zod'; import z from 'zod';
import type { ChildResponseItemType } from '../type'; import type { ChildResponseItemType } from '../type';
import { summarizeRuntimeNodeResponses } from '../../../utils'; import { summarizeRuntimeNodeResponses } from '../../../utils';
import { documentFileType } from '@fastgpt/global/common/file/constants';
const logger = getLogger(LogCategories.MODULE.AI.TOOL_CALL); const logger = getLogger(LogCategories.MODULE.AI.TOOL_CALL);
...@@ -26,11 +27,11 @@ export const ReadFileToolSchema: ChatCompletionTool = { ...@@ -26,11 +27,11 @@ export const ReadFileToolSchema: ChatCompletionTool = {
type: 'function', type: 'function',
function: { function: {
name: ReadFileTooData.id, name: ReadFileTooData.id,
description: '解析文件内容,获取文本。', description: `读取文档并返回文档内容,支持: ${documentFileType}`,
parameters: { parameters: {
type: 'object', type: 'object',
properties: { properties: {
ids: { type: 'array', items: { type: 'string' } } ids: { type: 'array', description: '需要读取的文档 id 列表', items: { type: 'string' } }
}, },
required: ['ids'] required: ['ids']
} }
......
...@@ -7,7 +7,6 @@ import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; ...@@ -7,7 +7,6 @@ import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { type ChatItemMiniType } from '@fastgpt/global/core/chat/type'; import { type ChatItemMiniType } from '@fastgpt/global/core/chat/type';
import { getNodeErrResponse } from '../utils'; import { getNodeErrResponse } from '../utils';
import { parseFileContentFromUrls } from '../../utils/file'; import { parseFileContentFromUrls } from '../../utils/file';
import { getUserFilesPrompt } from '../../../ai/llm/prompt';
import { sliceStrStartEnd } from '@fastgpt/global/common/string/tools'; import { sliceStrStartEnd } from '@fastgpt/global/common/string/tools';
type Props = ModuleDispatchProps<{ type Props = ModuleDispatchProps<{
...@@ -18,6 +17,20 @@ type Response = DispatchNodeResultType<{ ...@@ -18,6 +17,20 @@ type Response = DispatchNodeResultType<{
[NodeOutputKeyEnum.rawResponse]: { filename: string; url: string; text: string }[]; [NodeOutputKeyEnum.rawResponse]: { filename: string; url: string; text: string }[];
}>; }>;
/**
* 格式化 ReadFiles 节点已经读取出的文件正文。
*
* 这个输出会作为节点 text/toolResponse 传给后续节点,因此只描述“读取结果”,
* 不复用对话上传文件 reminder,避免混入“可通过 read_files 再读取”的工具说明。
*/
export const buildReadFilesOutputText = (
files: { id: string; name: string; content: string }[] = []
) => {
if (files.length === 0) return '';
return files.map((file) => `## ${file.name}\n${file.content}`).join('\n\n');
};
export const dispatchReadFiles = async (props: Props): Promise<Response> => { export const dispatchReadFiles = async (props: Props): Promise<Response> => {
const { const {
requestOrigin, requestOrigin,
...@@ -51,7 +64,7 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => { ...@@ -51,7 +64,7 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => {
content: item.content content: item.content
})); }));
const text = getUserFilesPrompt(files); const text = buildReadFilesOutputText(files);
const getPreviewResponse = files const getPreviewResponse = files
.map((item) => `## ${item.name}\n${sliceStrStartEnd(item.content, 1000, 1000)}`) .map((item) => `## ${item.name}\n${sliceStrStartEnd(item.content, 1000, 1000)}`)
...@@ -73,9 +86,7 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => { ...@@ -73,9 +86,7 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => {
})), })),
readFilesResult: getPreviewResponse readFilesResult: getPreviewResponse
}, },
[DispatchNodeResponseKeyEnum.toolResponse]: { [DispatchNodeResponseKeyEnum.toolResponse]: text
fileContent: text
}
}; };
} catch (error) { } catch (error) {
return getNodeErrResponse({ error }); return getNodeErrResponse({ error });
......
...@@ -38,6 +38,7 @@ export const formatUserQueryWithFiles = async ({ ...@@ -38,6 +38,7 @@ export const formatUserQueryWithFiles = async ({
{ {
id?: string; id?: string;
name: string; name: string;
url: string;
sandboxPath?: string; sandboxPath?: string;
content?: string; content?: string;
}[] }[]
......
...@@ -101,6 +101,7 @@ const getUserContextMessagesForTest = async ({ ...@@ -101,6 +101,7 @@ const getUserContextMessagesForTest = async ({
return { return {
chatHistories: context.chatHistories, chatHistories: context.chatHistories,
queryInput: context.queryInput, queryInput: context.queryInput,
fileUrlMap: context.fileUrlMap,
filesMap: context.filesMap, filesMap: context.filesMap,
...context.getCurrentMessages({ ...context.getCurrentMessages({
skillInfos, skillInfos,
...@@ -111,7 +112,7 @@ const getUserContextMessagesForTest = async ({ ...@@ -111,7 +112,7 @@ const getUserContextMessagesForTest = async ({
}; };
describe('buildAgentInputFilesPrompt', () => { describe('buildAgentInputFilesPrompt', () => {
it('generates file XML block for document files only', () => { it('generates file XML block with ids types and urls', () => {
const result = buildAgentInputFilesPrompt([ const result = buildAgentInputFilesPrompt([
{ {
id: 'current-0', id: 'current-0',
...@@ -139,15 +140,18 @@ describe('buildAgentInputFilesPrompt', () => { ...@@ -139,15 +140,18 @@ describe('buildAgentInputFilesPrompt', () => {
} }
]); ]);
expect(result).toContain('## 文件'); expect(result).toContain('## 对话文件');
expect(result).toContain('<id>current-0</id>'); expect(result).toContain('<id>current-0</id>');
expect(result).toContain('<name>guide.pdf</name>'); expect(result).toContain('<name>guide.pdf</name>');
expect(result).not.toContain('<id>current-1</id>'); expect(result).toContain('<type>file</type>');
expect(result).not.toContain('<id>current-2</id>'); expect(result).toContain('<url>/guide.pdf</url>');
expect(result).not.toContain('<id>current-3</id>'); expect(result).toContain('<id>current-1</id>');
expect(result).not.toContain('<type>image</type>'); expect(result).toContain('<id>current-2</id>');
expect(result).not.toContain('<type>audio</type>'); expect(result).toContain('<id>current-3</id>');
expect(result).not.toContain('<type>video</type>'); expect(result).toContain('<type>image</type>');
expect(result).toContain('<type>audio</type>');
expect(result).toContain('<type>video</type>');
expect(result).toContain('<url>/chart.png</url>');
}); });
it('escapes XML fields in file metadata', () => { it('escapes XML fields in file metadata', () => {
...@@ -162,6 +166,7 @@ describe('buildAgentInputFilesPrompt', () => { ...@@ -162,6 +166,7 @@ describe('buildAgentInputFilesPrompt', () => {
expect(result).toContain('<id>current-&amp;-&apos;&quot;-0</id>'); expect(result).toContain('<id>current-&amp;-&apos;&quot;-0</id>');
expect(result).toContain('<name>a&lt;b&gt;&amp;&quot;c&quot;&apos;d.pdf</name>'); expect(result).toContain('<name>a&lt;b&gt;&amp;&quot;c&quot;&apos;d.pdf</name>');
expect(result).toContain('<url>/guide.pdf</url>');
}); });
it('returns empty string when there are no files', () => { it('returns empty string when there are no files', () => {
...@@ -198,10 +203,10 @@ describe('buildAgentUserReminderInput', () => { ...@@ -198,10 +203,10 @@ describe('buildAgentUserReminderInput', () => {
expect(result).toContain('<system-reminder>'); expect(result).toContain('<system-reminder>');
expect(result).toContain('## 技能'); expect(result).toContain('## 技能');
expect(result).toContain('<path>/workspace/Skill/SKILL.md</path>'); expect(result).toContain('<path>/workspace/Skill/SKILL.md</path>');
expect(result.indexOf('## 技能')).toBeLessThan(result.indexOf('## 文件')); expect(result.indexOf('## 技能')).toBeLessThan(result.indexOf('## 对话文件'));
expect(result.indexOf('## 文件')).toBeLessThan(result.indexOf('## 知识库')); expect(result.indexOf('## 对话文件')).toBeLessThan(result.indexOf('## 知识库'));
expect(result.indexOf('## 知识库')).toBeLessThan(result.indexOf('## 背景信息')); expect(result.indexOf('## 知识库')).toBeLessThan(result.indexOf('## 背景信息'));
expect(result).toContain('## 文件'); expect(result).toContain('## 对话文件');
expect(result).toContain('## 知识库'); expect(result).toContain('## 知识库');
expect(result).toContain('<id>dataset_1</id>'); expect(result).toContain('<id>dataset_1</id>');
expect(result).toContain('## 背景信息'); expect(result).toContain('## 背景信息');
...@@ -265,7 +270,7 @@ describe('buildAgentUserReminderInput', () => { ...@@ -265,7 +270,7 @@ describe('buildAgentUserReminderInput', () => {
selectedDataset selectedDataset
}); });
expect(datasetOnly).toContain('## 知识库'); expect(datasetOnly).toContain('## 知识库');
expect(datasetOnly).not.toContain('## 文件'); expect(datasetOnly).not.toContain('## 对话文件');
expect(datasetOnly).not.toContain('当前时间'); expect(datasetOnly).not.toContain('当前时间');
}); });
...@@ -370,6 +375,11 @@ describe('useUserContext', () => { ...@@ -370,6 +375,11 @@ describe('useUserContext', () => {
'history_1-0': '/old.pdf', 'history_1-0': '/old.pdf',
'current_chat_item-0': '/current.pdf' 'current_chat_item-0': '/current.pdf'
}); });
expect(result.fileUrlMap).toEqual({
'history_1-0': '/old.pdf',
'current_chat_item-0': '/current.pdf',
'current_chat_item-1': '/current.png'
});
const { text: historyText } = chatValue2RuntimePrompt(result.rewrittenHistories[0].value); const { text: historyText } = chatValue2RuntimePrompt(result.rewrittenHistories[0].value);
const { text: currentText, files: currentFiles } = chatValue2RuntimePrompt( const { text: currentText, files: currentFiles } = chatValue2RuntimePrompt(
...@@ -383,7 +393,9 @@ describe('useUserContext', () => { ...@@ -383,7 +393,9 @@ describe('useUserContext', () => {
expect(currentText).toContain('## 背景信息'); expect(currentText).toContain('## 背景信息');
expect(currentText).toContain('当前 sandbox 工作目录: /workspace'); expect(currentText).toContain('当前 sandbox 工作目录: /workspace');
expect(currentText).toContain('<id>current_chat_item-0</id>'); expect(currentText).toContain('<id>current_chat_item-0</id>');
expect(currentText).not.toContain('<id>current_chat_item-1</id>'); expect(currentText).toContain('<id>current_chat_item-1</id>');
expect(currentText).toContain('<type>image</type>');
expect(currentText).toContain('<url>/current.png</url>');
expect(currentText).toContain('## 知识库'); expect(currentText).toContain('## 知识库');
expect(currentText).toContain('<description>后端读取到的知识库介绍</description>'); expect(currentText).toContain('<description>后端读取到的知识库介绍</description>');
expect(currentText).toContain('2026-05-14 10:00:00 Thursday'); expect(currentText).toContain('2026-05-14 10:00:00 Thursday');
...@@ -394,6 +406,12 @@ describe('useUserContext', () => { ...@@ -394,6 +406,12 @@ describe('useUserContext', () => {
name: 'current.pdf', name: 'current.pdf',
type: ChatFileTypeEnum.file, type: ChatFileTypeEnum.file,
url: '/current.pdf' url: '/current.pdf'
},
{
id: 'current_chat_item-1',
name: 'current.png',
type: ChatFileTypeEnum.image,
url: '/current.png'
} }
]); ]);
} }
...@@ -438,7 +456,7 @@ describe('useUserContext', () => { ...@@ -438,7 +456,7 @@ describe('useUserContext', () => {
const { text: historyText } = chatValue2RuntimePrompt(result.rewrittenHistories[0].value); const { text: historyText } = chatValue2RuntimePrompt(result.rewrittenHistories[0].value);
const { text: currentText } = chatValue2RuntimePrompt(result.currentUserMessage.value); const { text: currentText } = chatValue2RuntimePrompt(result.currentUserMessage.value);
expect(historyText).toContain('## 文件'); expect(historyText).toContain('## 对话文件');
expect(historyText).not.toContain('## 技能'); expect(historyText).not.toContain('## 技能');
expect(currentText).toContain('## 技能'); expect(currentText).toContain('## 技能');
expect(currentText).toContain('<path>/workspace/Report/SKILL.md</path>'); expect(currentText).toContain('<path>/workspace/Report/SKILL.md</path>');
...@@ -707,7 +725,7 @@ describe('useUserContext', () => { ...@@ -707,7 +725,7 @@ describe('useUserContext', () => {
); );
}); });
it('filters invalid urls and excludes image/audio/video files from agent context', async () => { it('filters invalid and data urls and keeps image audio video urls in agent context', async () => {
const dataImage = 'data:image/png;base64,AAAA'; const dataImage = 'data:image/png;base64,AAAA';
await runWithContextAsync( await runWithContextAsync(
{ {
...@@ -738,27 +756,45 @@ describe('useUserContext', () => { ...@@ -738,27 +756,45 @@ describe('useUserContext', () => {
}); });
expect(result.filesMap).toEqual({ expect(result.filesMap).toEqual({
'current_ai-2': '/doc.pdf' 'current_ai-0': '/doc.pdf'
});
expect(result.fileUrlMap).toEqual({
'current_ai-0': '/doc.pdf',
'current_ai-1': '/voice.mp3',
'current_ai-2': '/demo.mp4'
}); });
expect(result.currentFiles).toEqual([ expect(result.currentFiles).toEqual([
{ {
id: 'current_ai-2', id: 'current_ai-0',
name: 'doc.pdf', name: 'doc.pdf',
type: ChatFileTypeEnum.file, type: ChatFileTypeEnum.file,
url: '/doc.pdf' url: '/doc.pdf'
},
{
id: 'current_ai-1',
name: 'voice.mp3',
type: ChatFileTypeEnum.audio,
url: '/voice.mp3'
},
{
id: 'current_ai-2',
name: 'demo.mp4',
type: ChatFileTypeEnum.video,
url: '/demo.mp4'
} }
]); ]);
const { text } = chatValue2RuntimePrompt(result.currentUserMessage.value); const { text } = chatValue2RuntimePrompt(result.currentUserMessage.value);
expect(text).not.toContain('<id>current_ai-0</id>'); expect(text).toContain('<id>current_ai-0</id>');
expect(text).not.toContain('<id>current_ai-1</id>'); expect(text).toContain('<id>current_ai-1</id>');
expect(text).toContain('<id>current_ai-2</id>'); expect(text).toContain('<id>current_ai-2</id>');
expect(text).not.toContain('<id>current_ai-3</id>'); expect(text).not.toContain('<id>current_ai-3</id>');
expect(text).not.toContain('<id>current_ai-4</id>'); expect(text).not.toContain('<id>current_ai-4</id>');
expect(text).not.toContain('<type>image</type>'); expect(text).not.toContain('<type>image</type>');
expect(text).not.toContain('<type>audio</type>'); expect(text).toContain('<type>audio</type>');
expect(text).not.toContain('<type>video</type>'); expect(text).toContain('<type>video</type>');
expect(text).not.toContain('not-a-url'); expect(text).not.toContain('not-a-url');
expect(text).not.toContain('data:image');
expect(text).not.toContain('data:text/plain'); expect(text).not.toContain('data:text/plain');
} }
); );
...@@ -788,7 +824,7 @@ describe('useUserContext', () => { ...@@ -788,7 +824,7 @@ describe('useUserContext', () => {
expect(result.filesMap).toEqual({}); expect(result.filesMap).toEqual({});
const { text } = chatValue2RuntimePrompt(result.currentUserMessage.value); const { text } = chatValue2RuntimePrompt(result.currentUserMessage.value);
expect(text).not.toContain('## 文件'); expect(text).not.toContain('## 对话文件');
expect(text).toContain('分析文件'); expect(text).toContain('分析文件');
} }
); );
...@@ -939,7 +975,7 @@ describe('useUserContext', () => { ...@@ -939,7 +975,7 @@ describe('useUserContext', () => {
expect(result.rewrittenHistories[0]).toBe(result.chatHistories[0]); expect(result.rewrittenHistories[0]).toBe(result.chatHistories[0]);
const { text } = chatValue2RuntimePrompt(result.currentUserMessage.value); const { text } = chatValue2RuntimePrompt(result.currentUserMessage.value);
expect(text).not.toContain('## 文件'); expect(text).not.toContain('## 对话文件');
expect(text).toContain('当前问题'); expect(text).toContain('当前问题');
} }
); );
...@@ -976,7 +1012,7 @@ describe('useUserContext', () => { ...@@ -976,7 +1012,7 @@ describe('useUserContext', () => {
expect(files).toEqual([]); expect(files).toEqual([]);
expect(text).toContain('当前时间'); expect(text).toContain('当前时间');
expect(text).toContain('只问一个问题'); expect(text).toContain('只问一个问题');
expect(text).not.toContain('## 文件'); expect(text).not.toContain('## 对话文件');
} }
); );
}); });
......
...@@ -293,8 +293,10 @@ describe('dispatchRunAgent user context', () => { ...@@ -293,8 +293,10 @@ describe('dispatchRunAgent user context', () => {
content: expect.stringContaining('<id>current_ai_1-0</id>') content: expect.stringContaining('<id>current_ai_1-0</id>')
}) })
]); ]);
expect(loopInput.messages[0].content).toContain('<url>/old.pdf</url>');
expect(loopInput.messages[0].content).not.toContain('## 知识库'); expect(loopInput.messages[0].content).not.toContain('## 知识库');
expect(loopInput.messages[0].content).not.toContain('## 背景信息'); expect(loopInput.messages[0].content).not.toContain('## 背景信息');
expect(loopInput.messages[1].content).toContain('<url>/current.pdf</url>');
expect(loopInput.messages[1].content).toContain('## 知识库'); expect(loopInput.messages[1].content).toContain('## 知识库');
expect(loopInput.messages[1].content).toContain('## 背景信息'); expect(loopInput.messages[1].content).toContain('## 背景信息');
expect(loopInput.messages[1].content).toContain('当前问题'); expect(loopInput.messages[1].content).toContain('当前问题');
......
...@@ -334,6 +334,7 @@ describe('dispatchPiAgent user context', () => { ...@@ -334,6 +334,7 @@ describe('dispatchPiAgent user context', () => {
const prompt = agentPromptMock.mock.calls[0][0]; const prompt = agentPromptMock.mock.calls[0][0];
expect(prompt).toContain('<system-reminder>'); expect(prompt).toContain('<system-reminder>');
expect(prompt).toContain('<id>current_ai_1-0</id>'); expect(prompt).toContain('<id>current_ai_1-0</id>');
expect(prompt).toContain('<url>/current.pdf</url>');
expect(prompt).toContain('## 知识库'); expect(prompt).toContain('## 知识库');
expect(prompt).toContain('<id>dataset_1</id>'); expect(prompt).toContain('<id>dataset_1</id>');
expect(prompt).toContain('当前时间'); expect(prompt).toContain('当前时间');
......
import { beforeEach, describe, expect, it, vi } from 'vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest';
import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants';
import { getSubapps, getExecuteTool } from '@fastgpt/service/core/workflow/dispatch/ai/agent/utils'; import {
getSubapps,
getExecuteTool,
replaceAgentFileIdsWithUrls
} from '@fastgpt/service/core/workflow/dispatch/ai/agent/utils';
import { readFileTool } from '@fastgpt/service/core/workflow/dispatch/ai/agent/sub/file/utils'; import { readFileTool } from '@fastgpt/service/core/workflow/dispatch/ai/agent/sub/file/utils';
import { datasetSearchTool } from '@fastgpt/service/core/workflow/dispatch/ai/agent/sub/dataset/utils'; import { datasetSearchTool } from '@fastgpt/service/core/workflow/dispatch/ai/agent/sub/dataset/utils';
...@@ -8,11 +12,13 @@ const { ...@@ -8,11 +12,13 @@ const {
dispatchAgentDatasetSearchMock, dispatchAgentDatasetSearchMock,
dispatchAppMock, dispatchAppMock,
dispatchFileReadMock, dispatchFileReadMock,
dispatchToolMock,
getAgentRuntimeToolsMock getAgentRuntimeToolsMock
} = vi.hoisted(() => ({ } = vi.hoisted(() => ({
dispatchAgentDatasetSearchMock: vi.fn(), dispatchAgentDatasetSearchMock: vi.fn(),
dispatchAppMock: vi.fn(), dispatchAppMock: vi.fn(),
dispatchFileReadMock: vi.fn(), dispatchFileReadMock: vi.fn(),
dispatchToolMock: vi.fn(),
getAgentRuntimeToolsMock: vi.fn(async () => []) getAgentRuntimeToolsMock: vi.fn(async () => [])
})); }));
...@@ -24,6 +30,10 @@ vi.mock('@fastgpt/service/core/workflow/dispatch/ai/agent/sub/tool/utils', () => ...@@ -24,6 +30,10 @@ vi.mock('@fastgpt/service/core/workflow/dispatch/ai/agent/sub/tool/utils', () =>
getAgentRuntimeTools: getAgentRuntimeToolsMock getAgentRuntimeTools: getAgentRuntimeToolsMock
})); }));
vi.mock('@fastgpt/service/core/workflow/dispatch/ai/agent/sub/tool', () => ({
dispatchTool: dispatchToolMock
}));
vi.mock('@fastgpt/service/core/workflow/dispatch/ai/agent/sub/dataset', () => ({ vi.mock('@fastgpt/service/core/workflow/dispatch/ai/agent/sub/dataset', () => ({
dispatchAgentDatasetSearch: dispatchAgentDatasetSearchMock dispatchAgentDatasetSearch: dispatchAgentDatasetSearchMock
})); }));
...@@ -39,6 +49,30 @@ describe('Agent read_files tool protocol', () => { ...@@ -39,6 +49,30 @@ describe('Agent read_files tool protocol', () => {
getAgentRuntimeToolsMock.mockResolvedValue([]); getAgentRuntimeToolsMock.mockResolvedValue([]);
}); });
it('replaces exact agent file ids in user tool params with urls', () => {
const result = replaceAgentFileIdsWithUrls(
{
fileUrl: 'current-0',
nested: {
urls: ['current-0', 'current-1', 'keep']
},
text: 'please use current-0'
},
{
'current-0': 'https://files/current.pdf',
'current-1': 'https://files/image.png'
}
);
expect(result).toEqual({
fileUrl: 'https://files/current.pdf',
nested: {
urls: ['https://files/current.pdf', 'https://files/image.png', 'keep']
},
text: 'please use current-0'
});
});
it('exposes read_files with ids parameter', async () => { it('exposes read_files with ids parameter', async () => {
const { completionTools } = await getSubapps({ const { completionTools } = await getSubapps({
tmbId: 'tmb_1', tmbId: 'tmb_1',
...@@ -125,6 +159,92 @@ describe('Agent read_files tool protocol', () => { ...@@ -125,6 +159,92 @@ describe('Agent read_files tool protocol', () => {
); );
}); });
it('replaces agent file ids before dispatching user tools', async () => {
dispatchToolMock.mockResolvedValue({
response: 'tool response',
usages: [],
nodeResponse: {
moduleName: 'HTTP Tool'
}
});
const executeTool = getExecuteTool({
checkIsStopping: vi.fn(),
chatConfig: {},
runningUserInfo: {
teamId: 'team_1',
tmbId: 'tmb_1'
},
runningAppInfo: {
id: 'app_1'
},
chatId: 'chat_1',
uid: 'user_1',
variableState: {} as any,
externalProvider: {
openaiAccount: undefined
} as any,
lang: 'zh-CN',
requestOrigin: '',
mode: 'chat',
timezone: 'Asia/Shanghai',
retainDatasetCite: false,
maxRunTimes: 10,
workflowDispatchDeep: 0,
params: {
model: 'gpt-4'
},
stream: false,
getSubAppInfo: () => ({
name: 'HTTP Tool',
avatar: '',
toolDescription: ''
}),
getSubApp: () => ({
type: 'tool',
id: 'http-tool',
name: 'HTTP Tool',
avatar: '',
version: '1.0.0',
toolConfig: {},
params: {
fixedFile: 'current-0'
}
}),
completionTools: [],
fileUrlMap: {
'current-0': 'https://files/current.pdf',
'current-1': 'https://files/image.png'
},
filesMap: {}
} as any);
await executeTool({
callId: 'call_http_tool',
toolId: 'http-tool',
args: JSON.stringify({
fileUrl: 'current-1',
nested: {
list: ['current-0', 'keep']
},
text: 'please use current-0'
})
});
expect(dispatchToolMock).toHaveBeenCalledTimes(1);
expect(dispatchToolMock).toHaveBeenCalledWith(
expect.objectContaining({
params: {
fixedFile: 'https://files/current.pdf',
fileUrl: 'https://files/image.png',
nested: {
list: ['https://files/current.pdf', 'keep']
},
text: 'please use current-0'
}
})
);
});
it('exposes dataset search with query array parameter', async () => { it('exposes dataset search with query array parameter', async () => {
const { completionTools } = await getSubapps({ const { completionTools } = await getSubapps({
tmbId: 'tmb_1', tmbId: 'tmb_1',
......
...@@ -53,10 +53,13 @@ describe('dispatchReadFiles', () => { ...@@ -53,10 +53,13 @@ describe('dispatchReadFiles', () => {
}); });
const text = result.data?.[NodeOutputKeyEnum.text]; const text = result.data?.[NodeOutputKeyEnum.text];
expect(text).toContain('## a.pdf');
expect(text).toContain('Alpha'); expect(text).toContain('Alpha');
expect(text).toContain('## b.pdf');
expect(text).toContain('Beta'); expect(text).toContain('Beta');
expect(text).toContain('a.pdf'); expect(text).not.toContain('用户本次对话上传的文件');
expect(text).toContain('b.pdf'); expect(text).not.toContain('可通过 read_files');
expect(text).not.toContain('<file>');
expect(result.data?.[NodeOutputKeyEnum.rawResponse]).toEqual([ expect(result.data?.[NodeOutputKeyEnum.rawResponse]).toEqual([
{ filename: 'a.pdf', url: '/a.pdf', text: 'Alpha' }, { filename: 'a.pdf', url: '/a.pdf', text: 'Alpha' },
...@@ -73,9 +76,7 @@ describe('dispatchReadFiles', () => { ...@@ -73,9 +76,7 @@ describe('dispatchReadFiles', () => {
expect(nodeResponse.readFilesResult).toContain('## b.pdf'); expect(nodeResponse.readFilesResult).toContain('## b.pdf');
expect(nodeResponse.readFilesResult).toContain('Beta'); expect(nodeResponse.readFilesResult).toContain('Beta');
expect(result[DispatchNodeResponseKeyEnum.toolResponse]).toEqual({ expect(result[DispatchNodeResponseKeyEnum.toolResponse]).toBe(text);
fileContent: text
});
}); });
it('chatConfig 提供 maxFiles 和 customPdfParse 时按其值传入', async () => { it('chatConfig 提供 maxFiles 和 customPdfParse 时按其值传入', async () => {
...@@ -206,7 +207,7 @@ describe('dispatchReadFiles', () => { ...@@ -206,7 +207,7 @@ describe('dispatchReadFiles', () => {
const nodeResponse = result[DispatchNodeResponseKeyEnum.nodeResponse] as any; const nodeResponse = result[DispatchNodeResponseKeyEnum.nodeResponse] as any;
expect(nodeResponse.readFiles).toEqual([]); expect(nodeResponse.readFiles).toEqual([]);
expect(nodeResponse.readFilesResult).toBe(''); expect(nodeResponse.readFilesResult).toBe('');
expect(result[DispatchNodeResponseKeyEnum.toolResponse]).toEqual({ fileContent: '' }); expect(result[DispatchNodeResponseKeyEnum.toolResponse]).toBe('');
}); });
it('超大内容下预览仍按 sliceStrStartEnd 截断 (start/end 各 1000)', async () => { it('超大内容下预览仍按 sliceStrStartEnd 截断 (start/end 各 1000)', async () => {
......
...@@ -103,13 +103,14 @@ const createMockParseFileFn = ({ maxFiles = 20 }: { maxFiles?: number } = {}) => ...@@ -103,13 +103,14 @@ const createMockParseFileFn = ({ maxFiles = 20 }: { maxFiles?: number } = {}) =>
return rawTextBuffer return rawTextBuffer
? { ? {
name: rawTextBuffer.filename, name: rawTextBuffer.filename,
url,
content: rawTextBuffer.text content: rawTextBuffer.text
} }
: undefined; : undefined;
}) })
); );
return files.filter(Boolean) as { name: string; content: string }[]; return files.filter(Boolean) as { name: string; url: string; content: string }[];
}); });
const rewriteMessagesWithFileContent = async ({ const rewriteMessagesWithFileContent = async ({
...@@ -575,6 +576,7 @@ describe('formatUserQueryWithFiles', () => { ...@@ -575,6 +576,7 @@ describe('formatUserQueryWithFiles', () => {
{ {
id: 'file-1', id: 'file-1',
name: 'a.pdf', name: 'a.pdf',
url: '/a.pdf',
sandboxPath: 'user_files/a.pdf', sandboxPath: 'user_files/a.pdf',
content: 'Alpha' content: 'Alpha'
} }
...@@ -598,6 +600,7 @@ describe('formatUserQueryWithFiles', () => { ...@@ -598,6 +600,7 @@ describe('formatUserQueryWithFiles', () => {
expect(content).toContain('总结这个文件'); expect(content).toContain('总结这个文件');
expect(content).toContain('<id>file-1</id>'); expect(content).toContain('<id>file-1</id>');
expect(content).toContain('<name>a.pdf</name>'); expect(content).toContain('<name>a.pdf</name>');
expect(content).toContain('<url>/a.pdf</url>');
expect(content).toContain('<sandboxPath>user_files/a.pdf</sandboxPath>'); expect(content).toContain('<sandboxPath>user_files/a.pdf</sandboxPath>');
expect(content).toContain('<content>Alpha</content>'); expect(content).toContain('<content>Alpha</content>');
}); });
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment