Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit ab0b7456 by Archer Committed by GitHub
Options

fix: file url in model (#7060) 

* fix: file url in model

* fix: test
parent 1127c1cf
Showing with 77 additions and 78 deletions
document/content/self-host/upgrading/4-15/41504.mdx
......@@ -51,6 +51,8 @@ fastgpt-plugin:
## 🐛 修复
1. 模型获取多模态文件链接异常。
## 🛠️ 代码优化
1. 插件服务从旧 `runtime` 结构调整为 pnpm workspace monorepo，拆分为 HTTP 服务入口、领域模型、用例、API adapter、基础设施、SDK 和 CLI。
......
document/data/doc-last-modified.json
{
"content/faq/chat.en.mdx": "2026-04-26T21:08:47+08:00",
"content/faq/chat.mdx": "2026-04-26T21:08:47+08:00",
"content/faq/index.en.mdx": "2026-04-26T21:08:47+08:00",
"content/faq/index.mdx": "2026-04-26T21:08:47+08:00",
"content/faq/index.en.mdx": "2026-06-04T16:10:15+08:00",
"content/faq/index.mdx": "2026-06-04T16:10:15+08:00",
"content/guide/admin/sso.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/admin/sso.mdx": "2026-06-02T16:55:40+08:00",
"content/guide/admin/teamMode.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/admin/teamMode.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/evaluation.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/evaluation.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/faq.en.mdx": "2026-06-04T16:10:15+08:00",
"content/guide/build/faq.mdx": "2026-06-04T16:10:15+08:00",
"content/guide/build/general/ai_settings.en.mdx": "2026-05-08T18:08:04+08:00",
"content/guide/build/general/ai_settings.mdx": "2026-05-08T18:08:04+08:00",
"content/guide/build/general/chat_input_guide.en.mdx": "2026-05-07T15:06:40+08:00",
......@@ -85,6 +87,8 @@
"content/guide/dataset/collection_tags.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/dataset/dataset_engine.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/dataset/dataset_engine.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/dataset/faq.en.mdx": "2026-06-04T16:10:15+08:00",
"content/guide/dataset/faq.mdx": "2026-06-04T16:10:15+08:00",
"content/guide/dataset/rag.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/dataset/rag.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/dataset/third-party/api_dataset.en.mdx": "2026-05-07T15:06:40+08:00",
......@@ -137,12 +141,20 @@
"content/openapi/intro.mdx": "2026-05-29T21:03:14+08:00",
"content/openapi/share.en.mdx": "2026-04-26T21:08:47+08:00",
"content/openapi/share.mdx": "2026-04-26T21:08:47+08:00",
"content/plugin/index.en.mdx": "2026-06-04T16:10:15+08:00",
"content/plugin/index.mdx": "2026-06-04T16:10:15+08:00",
"content/plugin/intro.en.mdx": "2026-06-04T16:10:15+08:00",
"content/plugin/intro.mdx": "2026-06-04T16:10:15+08:00",
"content/plugin/model-presets.en.mdx": "2026-06-04T16:10:15+08:00",
"content/plugin/model-presets.mdx": "2026-06-04T16:10:15+08:00",
"content/plugin/system-tool-development.en.mdx": "2026-06-04T16:10:15+08:00",
"content/plugin/system-tool-development.mdx": "2026-06-04T16:10:15+08:00",
"content/self-host/config/env.en.mdx": "2026-05-27T12:17:46+08:00",
"content/self-host/config/env.mdx": "2026-05-27T12:17:46+08:00",
"content/self-host/config/json.en.mdx": "2026-05-25T11:21:30+08:00",
"content/self-host/config/json.mdx": "2026-05-25T11:21:30+08:00",
"content/self-host/config/model/intro.en.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/config/model/intro.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/config/model/intro.en.mdx": "2026-06-04T16:10:15+08:00",
"content/self-host/config/model/intro.mdx": "2026-06-04T16:10:15+08:00",
"content/self-host/config/model/minimax.en.mdx": "2026-06-03T10:40:17+08:00",
"content/self-host/config/model/minimax.mdx": "2026-06-03T10:40:17+08:00",
"content/self-host/config/model/siliconCloud.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -173,8 +185,8 @@
"content/self-host/deploy/sealos.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/design/dataset.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/design/dataset.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/design/design_plugin.en.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/design/design_plugin.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/design/design_plugin.en.mdx": "2026-06-04T16:10:15+08:00",
"content/self-host/design/design_plugin.mdx": "2026-06-04T16:10:15+08:00",
"content/self-host/dev.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/dev.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/index.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -263,7 +275,7 @@
"content/self-host/upgrading/4-15/41503.en.mdx": "2026-05-28T16:21:09+08:00",
"content/self-host/upgrading/4-15/41503.mdx": "2026-05-28T16:21:09+08:00",
"content/self-host/upgrading/4-15/41504.en.mdx": "2026-06-01T17:19:55+08:00",
"content/self-host/upgrading/4-15/41504.mdx": "2026-06-04T13:47:19+08:00",
"content/self-host/upgrading/4-15/41504.mdx": "2026-06-04T16:10:15+08:00",
"content/self-host/upgrading/outdated/40.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/40.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/41.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -340,8 +352,8 @@
"content/self-host/upgrading/outdated/4814.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/4815.en.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/upgrading/outdated/4815.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/upgrading/outdated/4816.en.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/upgrading/outdated/4816.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/upgrading/outdated/4816.en.mdx": "2026-06-04T16:10:15+08:00",
"content/self-host/upgrading/outdated/4816.mdx": "2026-06-04T16:10:15+08:00",
"content/self-host/upgrading/outdated/4817.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/4817.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/4818.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -404,6 +416,6 @@
"content/self-host/upgrading/outdated/499.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/upgrading/upgrade-intruction.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/upgrade-intruction.mdx": "2026-04-26T21:08:47+08:00",
"content/toc.en.mdx": "2026-06-01T17:19:55+08:00",
"content/toc.mdx": "2026-06-01T17:19:55+08:00"
"content/toc.en.mdx": "2026-06-04T16:10:15+08:00",
"content/toc.mdx": "2026-06-04T16:10:15+08:00"
}
\ No newline at end of file
packages/global/openapi/core/app/common/api.ts
......@@ -130,11 +130,14 @@ export const ListAppBodySchema = z
description: '父级应用/文件夹 ID，为空时查询根目录'
}),
type: z
.union([z.enum(AppTypeEnum), z.array(z.enum(AppTypeEnum))])
.preprocess(
(value) => (value === '' ? undefined : value),
z.union([z.enum(AppTypeEnum), z.array(z.enum(AppTypeEnum))]).optional()
)
.optional()
.meta({
example: AppTypeEnum.workflow,
description: '应用类型筛选，支持单个类型或类型数组'
description: '应用类型筛选，支持单个类型或类型数组；空字符串按全部类型处理'
}),
searchKey: z.string().optional().meta({
example: '客服',
......
packages/global/test/core/app/openapi.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { ListAppBodySchema } from '@fastgpt/global/openapi/core/app/common/api';
describe('ListAppBodySchema', () => {
it('should treat empty app type as unspecified type', () => {
expect(ListAppBodySchema.parse({ type: '' })).toEqual({});
});
it('should support single app type and app type array', () => {
expect(ListAppBodySchema.parse({ type: AppTypeEnum.workflow }).type).toBe(AppTypeEnum.workflow);
expect(
ListAppBodySchema.parse({ type: [AppTypeEnum.folder, AppTypeEnum.workflow] }).type
).toEqual([AppTypeEnum.folder, AppTypeEnum.workflow]);
});
});
packages/service/core/ai/llm/utils.ts
......@@ -15,7 +15,6 @@ import { axios } from '../../../common/api/axios';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import { i18nT } from '@fastgpt/global/common/i18n/utils';
import { getImageBase64 } from '../../../common/file/image/utils';
import { getS3ChatSource } from '../../../common/s3/sources/chat';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { getLogger, LogCategories } from '../../../common/logger';
import { serviceEnv } from '../../../env';
......@@ -194,23 +193,6 @@ export const loadRequestMessages = async ({
};
/**
* 通过内部 S3 key 生成当前服务可读取的聊天文件 URL。
* 读取失败时返回 undefined，由调用方决定是否回退原始 URL。
*/
const getS3FileUrl = async (key?: string) => {
if (!key) return;
try {
return (
await getS3ChatSource().createGetChatFileURL({
key,
external: false
})
).url;
} catch {}
};
/**
* 从 URL 下载二进制内容并转成 data URL。
* 用于供应商不能访问内部文件或模型配置要求 base64 的媒体输入。
*/
......@@ -286,19 +268,11 @@ export const loadRequestMessages = async ({
});
/**
* 将 FastGPT 内部的 S3 key 还原成可读取 URL。
* key 解析失败时回退原始 URL，避免外部可访问链接被误丢弃。
*/
const resolveMediaUrl = async ({ key, url }: { key?: string; url: string }) => {
return (await getS3FileUrl(key)) || url;
};
/**
* 判断媒体是否需要在服务端转 base64。
* key/本地路径只有服务端能读取；MULTIPLE_DATA_TO_BASE64 用于兼容不支持远程 URL 的模型。
* 本地路径只有服务端能读取；MULTIPLE_DATA_TO_BASE64 用于兼容不支持远程 URL 的模型。
*/
const shouldLoadMediaAsBase64 = ({ key, url }: { key?: string; url: string }) =>
!!key || url.startsWith('/') || serviceEnv.MULTIPLE_DATA_TO_BASE64;
const shouldLoadMediaAsBase64 = (url: string) =>
url.startsWith('/') || serviceEnv.MULTIPLE_DATA_TO_BASE64;
/**
* 仅从短文本中识别媒体 URL。普通文档 URL 仍作为文本保留，不在这里转成 LLM 媒体输入。
......@@ -360,7 +334,7 @@ export const loadRequestMessages = async ({
const normalizeImageContentPart = async (
item: Extract<ChatCompletionContentPart, { type: 'image_url' }>
): Promise<ChatCompletionContentPart | undefined> => {
const { key, ...imageItem } = item;
const { key: _key, ...imageItem } = item;
const imgUrl = imageItem.image_url.url;
if (imgUrl.startsWith('data:image/')) {
......@@ -368,10 +342,9 @@ export const loadRequestMessages = async ({
}
try {
if (shouldLoadMediaAsBase64({ key, url: imgUrl })) {
if (shouldLoadMediaAsBase64(imgUrl)) {
try {
const url = await resolveMediaUrl({ key, url: imgUrl });
const { completeBase64: base64 } = await getImageBase64(url);
const { completeBase64: base64 } = await getImageBase64(imgUrl);
return {
...imageItem,
......@@ -409,17 +382,17 @@ export const loadRequestMessages = async ({
const normalizeAudioContentPart = async (
item: Extract<ChatCompletionContentPart, { type: 'input_audio' }>
): Promise<ChatCompletionContentPart | undefined> => {
const { key, ...audioItem } = item;
const { key: _key, ...audioItem } = item;
const audioData = audioItem.input_audio.data;
if (audioData.startsWith('data:')) {
return audioItem;
}
if (!shouldLoadMediaAsBase64({ key, url: audioData })) {
if (!shouldLoadMediaAsBase64(audioData)) {
return audioItem;
}
const data = await loadUrlAsBase64Data(await resolveMediaUrl({ key, url: audioData }));
const data = await loadUrlAsBase64Data(audioData);
return {
...audioItem,
input_audio: {
......@@ -436,17 +409,17 @@ export const loadRequestMessages = async ({
const normalizeVideoContentPart = async (
item: Extract<ChatCompletionContentPart, { type: 'video_url' }>
): Promise<ChatCompletionContentPart | undefined> => {
const { key, ...videoItem } = item;
const { key: _key, ...videoItem } = item;
const videoUrl = videoItem.video_url.url;
if (videoUrl.startsWith('data:')) {
return videoItem;
}
if (!shouldLoadMediaAsBase64({ key, url: videoUrl })) {
if (!shouldLoadMediaAsBase64(videoUrl)) {
return videoItem;
}
const url = await loadUrlAsBase64Data(await resolveMediaUrl({ key, url: videoUrl }));
const url = await loadUrlAsBase64Data(videoUrl);
return {
...videoItem,
video_url: {
......@@ -462,15 +435,15 @@ export const loadRequestMessages = async ({
const normalizeFileUrlContentPart = async (
item: Extract<ChatCompletionContentPart, { type: 'file_url' }>
): Promise<ChatCompletionContentPart | undefined> => {
const { key, ...fileItem } = item;
const { key: _key, ...fileItem } = item;
const fileType = fileItem.fileType || getFileTypeFromUrl(fileItem.url);
// 上传文件会先以 FastGPT 内部的 file_url 存在，发给模型前需要转成供应商支持的
// input_audio / video_url。普通 file 当前不直接透传给 LLM。
if (fileType === 'audio' && useAudio) {
const fileUrl = await resolveMediaUrl({ key, url: fileItem.url });
const fileUrl = fileItem.url;
const filename = getFilenameFromUrl(fileUrl, fileItem.name);
const audioUrl = shouldLoadMediaAsBase64({ key, url: fileUrl })
const audioUrl = shouldLoadMediaAsBase64(fileUrl)
? await loadUrlAsBase64Data(fileUrl)
: fileUrl;
......@@ -478,8 +451,8 @@ export const loadRequestMessages = async ({
}
if (fileType === 'video' && useVideo) {
const fileUrl = await resolveMediaUrl({ key, url: fileItem.url });
const videoUrl = shouldLoadMediaAsBase64({ key, url: fileUrl })
const fileUrl = fileItem.url;
const videoUrl = shouldLoadMediaAsBase64(fileUrl)
? await loadUrlAsBase64Data(fileUrl)
: fileUrl;
......@@ -502,7 +475,7 @@ export const loadRequestMessages = async ({
/**
* 将 FastGPT 内部媒体 content part 归一化为供应商可消费的消息格式。
* 本地路径、S3 key、以及强制 base64 开关会在这里统一转换。
* 本地路径和强制 base64 开关会在这里统一转换；内部 key 只会被剥离，不参与处理。
*/
const normalizeMediaContentParts = async (content: ChatCompletionContentPart[]) => {
const normalized = await Promise.all(content.map((item) => normalizeMediaContentPart(item)));
......
packages/service/test/core/ai/llm/utils.test.ts
......@@ -6,8 +6,6 @@ import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/typ
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import { describe, expect, it, vi, beforeEach } from 'vitest';
const mockCreateGetChatFileURL = vi.hoisted(() => vi.fn());
// Mock external dependencies
vi.mock('@fastgpt/service/common/string/tiktoken/index', () => {
const countGptMessagesTokens = vi.fn();
......@@ -37,12 +35,6 @@ vi.mock('@fastgpt/service/common/api/axios', () => ({
}
}));
vi.mock('@fastgpt/service/common/s3/sources/chat', () => ({
getS3ChatSource: () => ({
createGetChatFileURL: mockCreateGetChatFileURL
})
}));
import { countGptMessagesTokens } from '@fastgpt/service/common/string/tiktoken/index';
import { getImageBase64 } from '@fastgpt/service/common/file/image/utils';
import { serviceEnv } from '@fastgpt/service/env';
......@@ -57,6 +49,7 @@ const originalMultipleDataToBase64 = serviceEnv.MULTIPLE_DATA_TO_BASE64;
describe('filterGPTMessageByMaxContext function tests', () => {
beforeEach(() => {
vi.clearAllMocks();
serviceEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64;
mockCountGptMessagesTokens.mockResolvedValue(10);
});
......@@ -643,13 +636,10 @@ describe('loadRequestMessages function tests', () => {
});
});
it('should normalize internal audio and video file_url to base64 content parts', async () => {
mockAxiosGet.mockResolvedValue({
data: Buffer.from('media bytes')
});
mockCreateGetChatFileURL
.mockResolvedValueOnce({ url: '/chat/audio.mp3' })
.mockResolvedValueOnce({ url: '/chat/video.mp4' });
it('should ignore file keys and normalize audio and video file_url from existing urls', async () => {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = false;
const audioUrl = 'https://cdn.example.com/preview/audio.mp3';
const videoUrl = 'https://cdn.example.com/preview/video.mp4';
const messages: ChatCompletionMessageParam[] = [
{
role: ChatCompletionRequestMessageRoleEnum.User,
......@@ -657,14 +647,14 @@ describe('loadRequestMessages function tests', () => {
{
type: 'file_url',
name: 'audio.mp3',
url: 'http://localhost:9000/preview/audio.mp3',
url: audioUrl,
fileType: 'audio',
key: 'chat/audio.mp3'
},
{
type: 'file_url',
name: 'video.mp4',
url: 'http://localhost:9000/preview/video.mp4',
url: videoUrl,
fileType: 'video',
key: 'chat/video.mp4'
},
......@@ -689,20 +679,22 @@ describe('loadRequestMessages function tests', () => {
expect.objectContaining({
type: 'input_audio',
input_audio: {
data: `data:;base64,${Buffer.from('media bytes').toString('base64')}`,
data: audioUrl,
format: 'mp3'
}
}),
expect.objectContaining({
type: 'video_url',
video_url: {
url: `data:;base64,${Buffer.from('media bytes').toString('base64')}`
url: videoUrl
}
}),
expect.objectContaining({ type: 'text', text: 'Analyze these files' })
])
);
expect(content.some((item: any) => item.type === 'file_url')).toBe(false);
expect(content.some((item: any) => item.key)).toBe(false);
expect(mockAxiosGet).not.toHaveBeenCalled();
});
it('should preserve legacy vision extraction when extractFiles is omitted', async () => {
......
pro @ 1ebfaac6
Subproject commit 35458b567a3b719987bdd4e9a18dc747a8b34495
Subproject commit 1ebfaac6a4e6faef2c01d4b6785add3debf7285f
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment