Commit ff77300f by YeYuheng Committed by GitHub

feat: add image dataset UI (#6932)

* feat: add image dataset ui

feat: address dataset image review comments

fix: restore dataset test panel styles

refactor: merge dataset data index update api

perf: data controler code

perf: review

* refactor: improve dataset detail input panels

* test: add dataset data service coverage

* fix: avoid dataset index update race

---------

Co-authored-by: archer <545436317@qq.com>
parent 45be8908
...@@ -11,6 +11,8 @@ description: 'FastGPT V4.15.0-beta2 更新说明' ...@@ -11,6 +11,8 @@ description: 'FastGPT V4.15.0-beta2 更新说明'
2. 增加 workflow zod 检查鲁棒性。 2. 增加 workflow zod 检查鲁棒性。
3. 禁用工作流无效连接模式。 3. 禁用工作流无效连接模式。
4. 增加工作流节点,名字超长适配。 4. 增加工作流节点,名字超长适配。
5. 知识库搜索测试交互。
6. 知识库数据编辑弹窗。
## 🐛 修复 ## 🐛 修复
......
...@@ -100,6 +100,7 @@ export const EmbeddingModelItemSchema = PriceTypeSchema.extend(BaseModelItemSche ...@@ -100,6 +100,7 @@ export const EmbeddingModelItemSchema = PriceTypeSchema.extend(BaseModelItemSche
maxToken: z.number(), // model max token maxToken: z.number(), // model max token
weight: z.number(), // training weight weight: z.number(), // training weight
hidden: z.boolean().optional(), // Disallow creation hidden: z.boolean().optional(), // Disallow creation
vision: z.boolean().optional(), // Support image embedding
normalization: z.boolean().optional(), // normalization processing normalization: z.boolean().optional(), // normalization processing
batchSize: z.number().optional(), // batch request size batchSize: z.number().optional(), // batch request size
defaultConfig: z.record(z.string(), z.any()).optional(), // post request config defaultConfig: z.record(z.string(), z.any()).optional(), // post request config
......
...@@ -13,7 +13,7 @@ export enum DatasetTypeEnum { ...@@ -13,7 +13,7 @@ export enum DatasetTypeEnum {
dingtalk = 'dingtalk' dingtalk = 'dingtalk'
} }
// @ts-ignore // @ts-expect-error ignore incomplete api dataset map
export const ApiDatasetTypeMap: Record< export const ApiDatasetTypeMap: Record<
`${DatasetTypeEnum}`, `${DatasetTypeEnum}`,
{ {
......
import { i18nT } from '../../../../web/i18n/utils'; import { i18nT } from '../../../../web/i18n/utils';
export enum DatasetDataIndexTypeEnum { export enum DatasetDataIndexTypeEnum {
default = 'default', default = 'default', // 默认的
custom = 'custom', summary = 'summary', // 摘要,系统生成
summary = 'summary', question = 'question', // 补全问题,系统生成
question = 'question', image = 'image', // 图片描述,系统生成
image = 'image' custom = 'custom'
} }
export const DatasetDataIndexMap: Record< export const DatasetDataIndexMap: Record<
......
...@@ -337,6 +337,7 @@ export const DatasetDataItemSchema = DatasetDataFieldSchema.extend({ ...@@ -337,6 +337,7 @@ export const DatasetDataItemSchema = DatasetDataFieldSchema.extend({
sourceId: z.string().optional().meta({ description: '来源 ID' }), sourceId: z.string().optional().meta({ description: '来源 ID' }),
chunkIndex: z.number().meta({ description: '块索引' }), chunkIndex: z.number().meta({ description: '块索引' }),
indexes: z.array(DatasetDataIndexItemSchema).meta({ description: '向量索引' }), indexes: z.array(DatasetDataIndexItemSchema).meta({ description: '向量索引' }),
imageDescMap: z.record(z.string(), z.string()).optional().meta({ description: '图片描述映射' }),
isOwner: z.boolean().meta({ description: '是否为 owner' }) isOwner: z.boolean().meta({ description: '是否为 owner' })
}); });
export type DatasetDataItemType = z.infer<typeof DatasetDataItemSchema>; export type DatasetDataItemType = z.infer<typeof DatasetDataItemSchema>;
......
...@@ -40,7 +40,7 @@ export function getSourceNameIcon({ ...@@ -40,7 +40,7 @@ export function getSourceNameIcon({
if (strIsLink(sourceId)) { if (strIsLink(sourceId)) {
return 'common/linkBlue'; return 'common/linkBlue';
} }
} catch (error) {} } catch {}
return 'file/fill/file'; return 'file/fill/file';
} }
...@@ -51,3 +51,5 @@ export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => { ...@@ -51,3 +51,5 @@ export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => {
if (mode === TrainingModeEnum.image) return data.length * 2; if (mode === TrainingModeEnum.image) return data.length * 2;
return data.length; return data.length;
}; };
export const isDatasetFileObjectKey = (url: string) => /^(temp|chat|dataset)\//i.test(url);
...@@ -215,6 +215,17 @@ export const DispatchNodeResponseSchema = z ...@@ -215,6 +215,17 @@ export const DispatchNodeResponseSchema = z
limit: z.number().optional().meta({ description: '限制' }), limit: z.number().optional().meta({ description: '限制' }),
searchMode: z.enum(DatasetSearchModeEnum).optional().meta({ description: '搜索模式' }), searchMode: z.enum(DatasetSearchModeEnum).optional().meta({ description: '搜索模式' }),
embeddingWeight: z.number().optional().meta({ description: '嵌入权重' }), embeddingWeight: z.number().optional().meta({ description: '嵌入权重' }),
filteredFileCount: z.number().optional().meta({ description: '过滤的非图片文件数量' }),
queryImages: z
.array(
z.object({
key: z.string().optional(),
url: z.string().optional(),
name: z.string().optional()
})
)
.optional()
.meta({ description: '参与知识库检索的图片' }),
rerankModel: z.string().optional().meta({ description: '重排模型' }), rerankModel: z.string().optional().meta({ description: '重排模型' }),
rerankWeight: z.number().optional().meta({ description: '重排权重' }), rerankWeight: z.number().optional().meta({ description: '重排权重' }),
reRankInputTokens: z.number().optional().meta({ description: '重排输入 token' }), reRankInputTokens: z.number().optional().meta({ description: '重排输入 token' }),
...@@ -359,7 +370,10 @@ export type DispatchNodeResponseType = Omit< ...@@ -359,7 +370,10 @@ export type DispatchNodeResponseType = Omit<
toolDetail?: DispatchNodeResponseType[]; toolDetail?: DispatchNodeResponseType[];
}; };
export type DispatchNodeResultType<T = {}, ERR = { [NodeOutputKeyEnum.errorText]?: string }> = { export type DispatchNodeResultType<
T = unknown,
ERR = { [NodeOutputKeyEnum.errorText]?: string }
> = {
[DispatchNodeResponseKeyEnum.answerText]?: string; [DispatchNodeResponseKeyEnum.answerText]?: string;
[DispatchNodeResponseKeyEnum.reasoningText]?: string; [DispatchNodeResponseKeyEnum.reasoningText]?: string;
[DispatchNodeResponseKeyEnum.skipHandleId]?: string[]; // skip some edge handle id [DispatchNodeResponseKeyEnum.skipHandleId]?: string[]; // skip some edge handle id
......
...@@ -315,15 +315,26 @@ export type CreateDatasetFolderBody = z.infer<typeof CreateDatasetFolderBodySche ...@@ -315,15 +315,26 @@ export type CreateDatasetFolderBody = z.infer<typeof CreateDatasetFolderBodySche
* API: 搜索测试 * API: 搜索测试
* Route: POST /api/core/dataset/searchTest * Route: POST /api/core/dataset/searchTest
* ============================================================================ */ * ============================================================================ */
export const SearchDatasetTestBodySchema = z.object({ export const SearchDatasetTestBodySchema = z
.object({
datasetId: ObjectIdSchema.meta({ datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05', example: '68ad85a7463006c963799a05',
description: '知识库 ID' description: '知识库 ID'
}), }),
text: z.string().meta({ text: z.string().optional().default('').meta({
example: 'FastGPT 是什么', example: 'FastGPT 是什么',
description: '搜索文本' description: '搜索文本'
}), }),
queryImageUrls: z
.array(z.string().min(1))
.max(10, '最多支持上传10张图片')
.optional()
.default([])
.meta({
example: ['temp/teamId/search-image.png'],
description:
'搜索测试图片临时 key,最多 10 张。需先调用 /api/core/dataset/file/presignSearchTestImage 获取预签名上传 URL 和 temp/${teamId}/... key,不支持直接传公网 URL、dataset key 或 chat key'
}),
similarity: z.number().optional().meta({ similarity: z.number().optional().meta({
example: 0.3, example: 0.3,
description: '最低相似度阈值' description: '最低相似度阈值'
...@@ -370,7 +381,10 @@ export const SearchDatasetTestBodySchema = z.object({ ...@@ -370,7 +381,10 @@ export const SearchDatasetTestBodySchema = z.object({
datasetDeepSearchBg: z.string().optional().meta({ datasetDeepSearchBg: z.string().optional().meta({
description: '深度搜索背景描述' description: '深度搜索背景描述'
}) })
}); })
.refine((data) => !!data.text.trim() || data.queryImageUrls.length > 0, {
message: 'text or queryImageUrls is required'
});
export type SearchDatasetTestBody = z.infer<typeof SearchDatasetTestBodySchema>; export type SearchDatasetTestBody = z.infer<typeof SearchDatasetTestBodySchema>;
export const SearchDatasetTestResponseSchema = z.object({ export const SearchDatasetTestResponseSchema = z.object({
......
...@@ -53,6 +53,50 @@ export type GetDatasetDataDetailResponse = z.infer<typeof GetDatasetDataDetailRe ...@@ -53,6 +53,50 @@ export type GetDatasetDataDetailResponse = z.infer<typeof GetDatasetDataDetailRe
* ============================================================================ */ * ============================================================================ */
export const UpdateDatasetDataBodySchema = UpdateDatasetDataPropsSchema; export const UpdateDatasetDataBodySchema = UpdateDatasetDataPropsSchema;
export type UpdateDatasetDataBody = z.infer<typeof UpdateDatasetDataBodySchema>; export type UpdateDatasetDataBody = z.infer<typeof UpdateDatasetDataBodySchema>;
export const UpdateDatasetDataResponseSchema = z.object({});
export type UpdateDatasetDataResponse = z.infer<typeof UpdateDatasetDataResponseSchema>;
const DatasetDataIndexContentSchema = z.object({
type: DatasetDataIndexItemSchema.shape.type.meta({
description: '索引类型'
}),
text: z.string().min(1).meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '索引文本'
})
});
/* ============================================================================
* API: 新增数据索引
* Route: POST /api/core/dataset/data/index/create
* ============================================================================ */
export const CreateDatasetDataIndexBodySchema = DatasetDataIndexContentSchema.extend({
dataId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
})
});
export type CreateDatasetDataIndexBody = z.infer<typeof CreateDatasetDataIndexBodySchema>;
export const DatasetDataIndexResponseSchema = z.object({
index: DatasetDataIndexItemSchema
});
export type DatasetDataIndexResponse = z.infer<typeof DatasetDataIndexResponseSchema>;
/* ============================================================================
* API: 更新数据索引
* Route: POST /api/core/dataset/data/index/update
* ============================================================================ */
export const UpdateDatasetDataIndexBodySchema = DatasetDataIndexContentSchema.extend({
dataId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
}),
indexDataId: z.string().meta({
example: '68ad85a7463006c963799a09',
description: '索引对应的 vectorDB ID'
})
});
export type UpdateDatasetDataIndexBody = z.infer<typeof UpdateDatasetDataIndexBodySchema>;
/* ============================================================================ /* ============================================================================
* API: 删除数据集数据 * API: 删除数据集数据
...@@ -66,6 +110,28 @@ export const DeleteDatasetDataQuerySchema = z.object({ ...@@ -66,6 +110,28 @@ export const DeleteDatasetDataQuerySchema = z.object({
}); });
export type DeleteDatasetDataQuery = z.infer<typeof DeleteDatasetDataQuerySchema>; export type DeleteDatasetDataQuery = z.infer<typeof DeleteDatasetDataQuerySchema>;
export const DeleteDatasetDataResponseSchema = z.literal('success');
export type DeleteDatasetDataResponse = z.infer<typeof DeleteDatasetDataResponseSchema>;
/* ============================================================================
* API: 删除数据索引
* Route: POST /api/core/dataset/data/index/delete
* ============================================================================ */
export const DeleteDatasetDataIndexBodySchema = z.object({
dataId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
}),
indexDataId: z.string().meta({
example: '68ad85a7463006c963799a09',
description: '索引对应的 vectorDB ID'
})
});
export type DeleteDatasetDataIndexBody = z.infer<typeof DeleteDatasetDataIndexBodySchema>;
export const DeleteDatasetDataIndexResponseSchema = z.object({});
export type DeleteDatasetDataIndexResponse = z.infer<typeof DeleteDatasetDataIndexResponseSchema>;
/* ============================================================================ /* ============================================================================
* API: 获取引用数据 * API: 获取引用数据
* Route: POST /api/core/dataset/data/getQuoteData * Route: POST /api/core/dataset/data/getQuoteData
......
...@@ -9,7 +9,15 @@ import { ...@@ -9,7 +9,15 @@ import {
InsertImagesBodySchema, InsertImagesBodySchema,
PushDataBodySchema, PushDataBodySchema,
GetDatasetDataListBodySchema, GetDatasetDataListBodySchema,
GetDatasetDataListResponseSchema GetDatasetDataListResponseSchema,
UpdateDatasetDataResponseSchema,
CreateDatasetDataIndexBodySchema,
DatasetDataIndexResponseSchema,
UpdateDatasetDataIndexBodySchema,
DeleteDatasetDataIndexBodySchema,
DeleteDatasetDataIndexResponseSchema,
GetDatasetDataDetailResponseSchema,
DeleteDatasetDataResponseSchema
} from './api'; } from './api';
export const DatasetDataPath: OpenAPIPath = { export const DatasetDataPath: OpenAPIPath = {
...@@ -47,7 +55,12 @@ export const DatasetDataPath: OpenAPIPath = { ...@@ -47,7 +55,12 @@ export const DatasetDataPath: OpenAPIPath = {
}, },
responses: { responses: {
200: { 200: {
description: '成功返回数据详情' description: '成功返回数据详情',
content: {
'application/json': {
schema: GetDatasetDataDetailResponseSchema
}
}
} }
} }
} }
...@@ -67,7 +80,87 @@ export const DatasetDataPath: OpenAPIPath = { ...@@ -67,7 +80,87 @@ export const DatasetDataPath: OpenAPIPath = {
}, },
responses: { responses: {
200: { 200: {
description: '更新成功' description: '更新成功',
content: {
'application/json': {
schema: UpdateDatasetDataResponseSchema
}
}
}
}
}
},
'/core/dataset/data/index/create': {
post: {
summary: '新增数据索引',
description: '新增指定数据下的单个自定义索引,不覆盖其它索引',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: CreateDatasetDataIndexBodySchema
}
}
},
responses: {
200: {
description: '保存成功',
content: {
'application/json': {
schema: DatasetDataIndexResponseSchema
}
}
}
}
}
},
'/core/dataset/data/index/update': {
post: {
summary: '更新数据索引',
description: '更新指定数据下的单个自定义索引,不覆盖其它索引',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: UpdateDatasetDataIndexBodySchema
}
}
},
responses: {
200: {
description: '更新成功',
content: {
'application/json': {
schema: DatasetDataIndexResponseSchema
}
}
}
}
}
},
'/core/dataset/data/index/delete': {
post: {
summary: '删除数据索引',
description: '删除指定数据下的单个索引,不触发整条数据重建',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: DeleteDatasetDataIndexBodySchema
}
}
},
responses: {
200: {
description: '删除成功',
content: {
'application/json': {
schema: DeleteDatasetDataIndexResponseSchema
}
}
} }
} }
} }
...@@ -83,7 +176,12 @@ export const DatasetDataPath: OpenAPIPath = { ...@@ -83,7 +176,12 @@ export const DatasetDataPath: OpenAPIPath = {
}, },
responses: { responses: {
200: { 200: {
description: '删除成功' description: '删除成功',
content: {
'application/json': {
schema: DeleteDatasetDataResponseSchema
}
}
} }
} }
} }
......
...@@ -2,6 +2,7 @@ import { z } from 'zod'; ...@@ -2,6 +2,7 @@ import { z } from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo'; import { ObjectIdSchema } from '../../../../common/type/mongo';
import { DatasetSourceReadTypeEnum } from '../../../../core/dataset/constants'; import { DatasetSourceReadTypeEnum } from '../../../../core/dataset/constants';
import { ChunkSettingsSchema } from '../../../../core/dataset/type'; import { ChunkSettingsSchema } from '../../../../core/dataset/type';
import { CreatePostPresignedUrlResponseSchema } from '../../../../common/file/s3/type';
/* ============================================================================ /* ============================================================================
* API: 预览文件分块 * API: 预览文件分块
...@@ -68,3 +69,73 @@ export const PresignDatasetFilePostUrlBodySchema = z.object({ ...@@ -68,3 +69,73 @@ export const PresignDatasetFilePostUrlBodySchema = z.object({
}) })
}); });
export type PresignDatasetFilePostUrlBody = z.infer<typeof PresignDatasetFilePostUrlBodySchema>; export type PresignDatasetFilePostUrlBody = z.infer<typeof PresignDatasetFilePostUrlBodySchema>;
export const PresignDatasetFilePostUrlResponseSchema = CreatePostPresignedUrlResponseSchema.meta({
description: 'S3 预签名上传 URL 及相关头信息'
});
export type PresignDatasetFilePostUrlResponse = z.infer<
typeof PresignDatasetFilePostUrlResponseSchema
>;
/* ============================================================================
* API: 获取搜索测试图片上传预签名 URL
* Route: POST /api/core/dataset/file/presignSearchTestImage
* Method: POST
* Description: 获取用于知识库搜索测试图片上传的临时预签名 URL,上传对象 3 小时后过期
* Tags: ['Dataset', 'File', 'Write']
* ============================================================================ */
export const PresignSearchTestImageBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
filename: z.string().min(1).meta({
example: 'demo.png',
description: '待上传图片文件名'
})
});
export type PresignSearchTestImageBody = z.infer<typeof PresignSearchTestImageBodySchema>;
export const PresignSearchTestImageResponseSchema = CreatePostPresignedUrlResponseSchema.meta({
description: '搜索测试图片上传预签名 URL、临时 key 和预览 URL'
});
export type PresignSearchTestImageResponse = z.infer<typeof PresignSearchTestImageResponseSchema>;
/* ============================================================================
* API: 获取搜索测试图片预览 URL
* Route: POST /api/core/dataset/file/getSearchTestImagePreviewUrls
* Method: POST
* Description: 根据搜索测试历史中的临时图片 key 重新生成短期预览 URL
* Tags: ['Dataset', 'File', 'Read']
* ============================================================================ */
export const GetSearchTestImagePreviewUrlsBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
keys: z
.array(z.string().min(1))
.max(10)
.meta({
example: ['temp/teamId/demo.png'],
description: '搜索测试图片临时 S3 key 列表,最多 10 个'
})
});
export type GetSearchTestImagePreviewUrlsBody = z.infer<
typeof GetSearchTestImagePreviewUrlsBodySchema
>;
export const GetSearchTestImagePreviewUrlsResponseSchema = z.array(
z.object({
key: z.string().meta({
example: 'temp/teamId/demo.png',
description: '临时图片 S3 key'
}),
previewUrl: z.string().meta({
description: '用于前端缩略图展示的临时预览 URL'
})
})
);
export type GetSearchTestImagePreviewUrlsResponse = z.infer<
typeof GetSearchTestImagePreviewUrlsResponseSchema
>;
import type { OpenAPIPath } from '../../../type'; import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag'; import { TagsMap } from '../../../tag';
import { import {
GetSearchTestImagePreviewUrlsBodySchema,
GetSearchTestImagePreviewUrlsResponseSchema,
GetPreviewChunksBodySchema, GetPreviewChunksBodySchema,
GetPreviewChunksResponseSchema, GetPreviewChunksResponseSchema,
PresignDatasetFilePostUrlBodySchema PresignDatasetFilePostUrlBodySchema,
PresignDatasetFilePostUrlResponseSchema,
PresignSearchTestImageBodySchema,
PresignSearchTestImageResponseSchema
} from './api'; } from './api';
import { CreatePostPresignedUrlResponseSchema } from '../../../../common/file/s3/type';
export const DatasetFilePath: OpenAPIPath = { export const DatasetFilePath: OpenAPIPath = {
'/core/dataset/file/getPreviewChunks': { '/core/dataset/file/getPreviewChunks': {
...@@ -49,7 +53,55 @@ export const DatasetFilePath: OpenAPIPath = { ...@@ -49,7 +53,55 @@ export const DatasetFilePath: OpenAPIPath = {
description: '成功返回预签名上传 URL、key、请求头和最大文件大小', description: '成功返回预签名上传 URL、key、请求头和最大文件大小',
content: { content: {
'application/json': { 'application/json': {
schema: CreatePostPresignedUrlResponseSchema schema: PresignDatasetFilePostUrlResponseSchema
}
}
}
}
}
},
'/core/dataset/file/presignSearchTestImage': {
post: {
summary: '获取搜索测试图片上传预签名 URL',
description: '获取搜索测试图片上传预签名 URL,仅支持图片文件,上传对象 3 小时后过期',
tags: [TagsMap.datasetFile],
requestBody: {
content: {
'application/json': {
schema: PresignSearchTestImageBodySchema
}
}
},
responses: {
200: {
description: '成功返回临时图片上传 URL、key 和缩略图预览 URL',
content: {
'application/json': {
schema: PresignSearchTestImageResponseSchema
}
}
}
}
}
},
'/core/dataset/file/getSearchTestImagePreviewUrls': {
post: {
summary: '获取搜索测试图片预览 URL',
description: '根据搜索测试历史中的临时图片 key 重新生成短期预览 URL',
tags: [TagsMap.datasetFile],
requestBody: {
content: {
'application/json': {
schema: GetSearchTestImagePreviewUrlsBodySchema
}
}
},
responses: {
200: {
description: '成功返回临时图片 key 和缩略图预览 URL 列表',
content: {
'application/json': {
schema: GetSearchTestImagePreviewUrlsResponseSchema
} }
} }
} }
......
...@@ -183,7 +183,8 @@ export const DatasetPath: OpenAPIPath = { ...@@ -183,7 +183,8 @@ export const DatasetPath: OpenAPIPath = {
'/core/dataset/searchTest': { '/core/dataset/searchTest': {
post: { post: {
summary: '搜索测试', summary: '搜索测试',
description: '对知识库执行搜索测试,支持多种搜索模式、重排序和问题扩展', description:
'对知识库执行搜索测试,支持多种搜索模式、重排序、问题扩展和临时图片 key 检索。图片检索需先调用 /core/dataset/file/presignSearchTestImage 获取预签名上传 URL 和 temp/${teamId}/... key',
tags: [TagsMap.datasetCommon], tags: [TagsMap.datasetCommon],
requestBody: { requestBody: {
content: { content: {
......
...@@ -5,7 +5,7 @@ import { serviceEnv } from '../../env'; ...@@ -5,7 +5,7 @@ import { serviceEnv } from '../../env';
const logger = getLogger(LogCategories.INFRA.MONGO); const logger = getLogger(LogCategories.INFRA.MONGO);
const maxConnecting = Math.max(30, serviceEnv.DB_MAX_LINK); const maxConnecting = Math.max(5, serviceEnv.DB_MAX_LINK);
/** /**
* connect MongoDB and init data * connect MongoDB and init data
...@@ -50,7 +50,7 @@ export async function connectMongo(props: { ...@@ -50,7 +50,7 @@ export async function connectMongo(props: {
bufferCommands: true, bufferCommands: true,
maxConnecting: maxConnecting, // 最大连接数: 防止连接数过多时无法满足需求 maxConnecting: maxConnecting, // 最大连接数: 防止连接数过多时无法满足需求
maxPoolSize: maxConnecting, // 最大连接池大小: 防止连接池过大时无法满足需求 maxPoolSize: maxConnecting, // 最大连接池大小: 防止连接池过大时无法满足需求
minPoolSize: 20, // 最小连接数: 20,防止连接数过少时无法满足需求 minPoolSize: 1,
connectTimeoutMS: 60000, // 连接超时: 60秒,防止连接失败时长时间阻塞 connectTimeoutMS: 60000, // 连接超时: 60秒,防止连接失败时长时间阻塞
waitQueueTimeoutMS: 60000, // 等待队列超时: 60秒,防止等待队列长时间阻塞 waitQueueTimeoutMS: 60000, // 等待队列超时: 60秒,防止等待队列长时间阻塞
socketTimeoutMS: 60000, // Socket 超时: 60秒,防止Socket连接失败时长时间阻塞 socketTimeoutMS: 60000, // Socket 超时: 60秒,防止Socket连接失败时长时间阻塞
......
...@@ -95,6 +95,7 @@ ...@@ -95,6 +95,7 @@
"model.vision": "Vision model", "model.vision": "Vision model",
"model.vision_tag": "Vision", "model.vision_tag": "Vision",
"model.vision_tip": "If the model supports image recognition, turn on this switch.", "model.vision_tip": "If the model supports image recognition, turn on this switch.",
"model.embedding_vision_tip": "Enable this when the embedding model can accept image input for image vector indexing and image search.",
"model.voices": "voice role", "model.voices": "voice role",
"model.voices_tip": "Configure multiple through an array, for example:\n\n[\n {\n \"label\": \"Alloy\",\n \"value\": \"alloy\"\n },\n {\n \"label\": \"Echo\",\n \"value\": \"echo\"\n }\n]", "model.voices_tip": "Configure multiple through an array, for example:\n\n[\n {\n \"label\": \"Alloy\",\n \"value\": \"alloy\"\n },\n {\n \"label\": \"Echo\",\n \"value\": \"echo\"\n }\n]",
"model_provider": "Model Provider", "model_provider": "Model Provider",
......
...@@ -81,7 +81,7 @@ ...@@ -81,7 +81,7 @@
"view_chart": "Chart", "view_chart": "Chart",
"view_table": "Table", "view_table": "Table",
"vlm_model": "Vlm", "vlm_model": "Vlm",
"vlm_model_tip": "Used to generate additional indexing of images in a document in the knowledge base", "vlm_model_tip": "Automatically labels images in documents and generates text descriptions to assist text retrieval",
"volunme_of_failed_calls": "Error amount", "volunme_of_failed_calls": "Error amount",
"waiting_test": "Waiting for testing" "waiting_test": "Waiting for testing"
} }
...@@ -240,6 +240,8 @@ ...@@ -240,6 +240,8 @@
"core.ai.Prompt": "Prompt", "core.ai.Prompt": "Prompt",
"core.ai.Support tool": "Tool call", "core.ai.Support tool": "Tool call",
"core.ai.model.Dataset Agent Model": "File read model", "core.ai.model.Dataset Agent Model": "File read model",
"core.ai.model.multimodal": "Multimodal",
"core.ai.model.multimodal_tip": "Multimodal embedding models can generate vectors for images.",
"core.ai.model.Vector Model": "Index model", "core.ai.model.Vector Model": "Index model",
"core.ai.model.doc_index_and_dialog": "Document Index & Dialog Index", "core.ai.model.doc_index_and_dialog": "Document Index & Dialog Index",
"core.app.Api request": "API Request", "core.app.Api request": "API Request",
...@@ -458,7 +460,7 @@ ...@@ -458,7 +460,7 @@
"core.dataset.data.Search data placeholder": "Search Related Data", "core.dataset.data.Search data placeholder": "Search Related Data",
"core.dataset.data.Updated": "Updated", "core.dataset.data.Updated": "Updated",
"core.dataset.data.group": " Groups", "core.dataset.data.group": " Groups",
"core.dataset.embedding model tip": "The index model can convert natural language into vectors for semantic search.\nNote that different index models cannot be used together. Once an index model is selected, it cannot be changed.", "core.dataset.embedding model tip": "The index model converts knowledge base content into vectors for semantic search. Note that knowledge bases using different index models cannot be queried together. Switching the index model requires rebuilding all vector indexes, so choose carefully.",
"core.dataset.error.Data not found": "Data Not Found or Deleted", "core.dataset.error.Data not found": "Data Not Found or Deleted",
"core.dataset.error.Start Sync Failed": "Failed to Start Sync", "core.dataset.error.Start Sync Failed": "Failed to Start Sync",
"core.dataset.error.unExistDataset": "The knowledge base does not exist", "core.dataset.error.unExistDataset": "The knowledge base does not exist",
...@@ -522,7 +524,14 @@ ...@@ -522,7 +524,14 @@
"core.dataset.test.Test Result": "Test Result", "core.dataset.test.Test Result": "Test Result",
"core.dataset.test.Test Text": "Single Text Test", "core.dataset.test.Test Text": "Single Text Test",
"core.dataset.test.Test Text Placeholder": "Enter the text to be tested", "core.dataset.test.Test Text Placeholder": "Enter the text to be tested",
"core.dataset.test.image_expired": "Image expired",
"core.dataset.test.image_search_disabled_tip": "Configure an image understanding model or multimodal embedding model first.",
"core.dataset.test.image_token": "[Image]",
"core.dataset.test.input_title": "Test input",
"core.dataset.test.max_images_tip": "Up to 10 images are supported",
"core.dataset.test.search_config": "Search configuration",
"core.dataset.test.Test params": "Test Parameters", "core.dataset.test.Test params": "Test Parameters",
"core.dataset.test.upload_image": "Upload image",
"core.dataset.test.delete test history": "Delete This Test Result", "core.dataset.test.delete test history": "Delete This Test Result",
"core.dataset.test.test history": "Test History", "core.dataset.test.test history": "Test History",
"core.dataset.test.test result placeholder": "Test results will be displayed here", "core.dataset.test.test result placeholder": "Test results will be displayed here",
...@@ -727,6 +736,7 @@ ...@@ -727,6 +736,7 @@
"dataset.data.Index Placeholder": "Enter Index Text Content", "dataset.data.Index Placeholder": "Enter Index Text Content",
"dataset.data.Input Success Tip": "Data Imported Successfully", "dataset.data.Input Success Tip": "Data Imported Successfully",
"dataset.data.Update Success Tip": "Data Updated Successfully", "dataset.data.Update Success Tip": "Data Updated Successfully",
"dataset.data.Update Index Timeout Tip": "Updating indexes timed out. Please refresh the data later to check the result.",
"dataset.data.edit.Index": "Data Index ({{amount}})", "dataset.data.edit.Index": "Data Index ({{amount}})",
"dataset.dataset_name": "Dataset Name", "dataset.dataset_name": "Dataset Name",
"dataset.deleteFolderTips": "Confirm to Delete This Folder and All Its Contained Datasets? Data Cannot Be Recovered After Deletion, Please Confirm!", "dataset.deleteFolderTips": "Confirm to Delete This Folder and All Its Contained Datasets? Data Cannot Be Recovered After Deletion, Please Confirm!",
......
...@@ -36,6 +36,8 @@ ...@@ -36,6 +36,8 @@
"common.error.unKnow": "Unknown error", "common.error.unKnow": "Unknown error",
"common_dataset": "General Dataset", "common_dataset": "General Dataset",
"common_dataset_desc": "Building a knowledge base by importing files, web page links, or manual entry", "common_dataset_desc": "Building a knowledge base by importing files, web page links, or manual entry",
"create_dataset_title": "Create {{name}}",
"dataset_name_placeholder": "Give the knowledge base a name",
"confirm_delete_collection": "Confirm to delete {{num }} files?", "confirm_delete_collection": "Confirm to delete {{num }} files?",
"confirm_import_images": "Total {{num}} | Confirm create", "confirm_import_images": "Total {{num}} | Confirm create",
"confirm_to_rebuild_embedding_tip": "This will re-vectorize all data in the knowledge base. It may take a while depending on the data volume. Proceed?", "confirm_to_rebuild_embedding_tip": "This will re-vectorize all data in the knowledge base. It may take a while depending on the data volume. Proceed?",
...@@ -86,8 +88,14 @@ ...@@ -86,8 +88,14 @@
"file_model_function_tip": "Used for QA generation, auto-indexing, and other AI-powered data processing.", "file_model_function_tip": "Used for QA generation, auto-indexing, and other AI-powered data processing.",
"filename": "Filename", "filename": "Filename",
"folder_dataset": "Folder", "folder_dataset": "Folder",
"generate_index": "Update index",
"image_auto_parse": "Automatic image indexing", "image_auto_parse": "Automatic image indexing",
"image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes", "image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes",
"image_auto_parse_tip_commercial": "Upgrade to the commercial edition to use this feature",
"image_auto_parse_tip_multimodal_with_vlm": "Generate image vector indexes and text description indexes for document images to support image search",
"image_auto_parse_tip_multimodal_without_vlm": "Use a multimodal model to generate image vector indexes and support image search",
"image_auto_parse_tip_vlm_only": "Use VLM to automatically label document images and generate text description indexes",
"image_auto_parse_tip_no_vlm_or_multimodal": "Configure an image understanding model or switch to a multimodal vector model before enabling this",
"images_creating": "Creating", "images_creating": "Creating",
"immediate_sync": "Immediate Synchronization", "immediate_sync": "Immediate Synchronization",
"import_confirm": "Start import", "import_confirm": "Start import",
...@@ -186,6 +194,7 @@ ...@@ -186,6 +194,7 @@
"uploading_progress": "Uploading: {{num}}%", "uploading_progress": "Uploading: {{num}}%",
"vector_model_max_tokens_tip": "Each chunk of data has a maximum length of 3000 tokens", "vector_model_max_tokens_tip": "Each chunk of data has a maximum length of 3000 tokens",
"vllm_model": "Image understanding model", "vllm_model": "Image understanding model",
"vllm_model_tip": "Automatically labels images in documents and generates text descriptions to assist text retrieval",
"website_dataset": "Web sync", "website_dataset": "Web sync",
"website_dataset_desc": "Build knowledge base by crawling web page data in batches", "website_dataset_desc": "Build knowledge base by crawling web page data in batches",
"website_info": "Website Information", "website_info": "Website Information",
......
{ {
"Image_Preview": "Picture preview", "Image_Preview": "Picture preview",
"Image_dataset_requires_VLM_model_to_be_configured": "The image dataset needs to be configured with the image understanding model (VLM) to be used. Please add a model that supports image understanding in the model configuration first.", "Image_dataset_requires_VLM_model_to_be_configured": "Image datasets require an image understanding model or a multimodal index model. Please configure a matching model first.",
"click_to_view_raw_source": "Click to View Original Source", "click_to_view_raw_source": "Click to View Original Source",
"file_name": "Filename", "file_name": "Filename",
"file_size": "Filesize", "file_size": "Filesize",
......
...@@ -95,6 +95,7 @@ ...@@ -95,6 +95,7 @@
"model.vision": "支持图片识别", "model.vision": "支持图片识别",
"model.vision_tag": "视觉", "model.vision_tag": "视觉",
"model.vision_tip": "如果模型支持图片识别,则打开该开关。", "model.vision_tip": "如果模型支持图片识别,则打开该开关。",
"model.embedding_vision_tip": "开启后该索引模型可接收图片输入,并用于图片向量索引和图搜图。",
"model.voices": "声音角色", "model.voices": "声音角色",
"model.voices_tip": "通过一个数组配置多个,例如:\n[\n {\n \"label\": \"Alloy\",\n \"value\": \"alloy\"\n },\n {\n \"label\": \"Echo\",\n \"value\": \"echo\"\n }\n]", "model.voices_tip": "通过一个数组配置多个,例如:\n[\n {\n \"label\": \"Alloy\",\n \"value\": \"alloy\"\n },\n {\n \"label\": \"Echo\",\n \"value\": \"echo\"\n }\n]",
"model_provider": "模型提供商", "model_provider": "模型提供商",
......
...@@ -81,7 +81,7 @@ ...@@ -81,7 +81,7 @@
"view_chart": "图表", "view_chart": "图表",
"view_table": "表格", "view_table": "表格",
"vlm_model": "图片理解模型", "vlm_model": "图片理解模型",
"vlm_model_tip": "用于知识库中对文档中的图片进行额外的索引生成", "vlm_model_tip": "自动标注文档里的图片并生成文本描述,辅助文本检索",
"volunme_of_failed_calls": "调用失败量", "volunme_of_failed_calls": "调用失败量",
"waiting_test": "等待测试" "waiting_test": "等待测试"
} }
...@@ -240,6 +240,8 @@ ...@@ -240,6 +240,8 @@
"core.ai.Prompt": "提示词", "core.ai.Prompt": "提示词",
"core.ai.Support tool": "工具调用", "core.ai.Support tool": "工具调用",
"core.ai.model.Dataset Agent Model": "文本理解模型", "core.ai.model.Dataset Agent Model": "文本理解模型",
"core.ai.model.multimodal": "多模态",
"core.ai.model.multimodal_tip": "多模态索引模型可以给图片生成向量。",
"core.ai.model.Vector Model": "索引模型", "core.ai.model.Vector Model": "索引模型",
"core.ai.model.doc_index_and_dialog": "文档索引 & 对话索引", "core.ai.model.doc_index_and_dialog": "文档索引 & 对话索引",
"core.app.Api request": "API 访问", "core.app.Api request": "API 访问",
...@@ -458,7 +460,7 @@ ...@@ -458,7 +460,7 @@
"core.dataset.data.Search data placeholder": "搜索相关数据", "core.dataset.data.Search data placeholder": "搜索相关数据",
"core.dataset.data.Updated": "已更新", "core.dataset.data.Updated": "已更新",
"core.dataset.data.group": "组", "core.dataset.data.group": "组",
"core.dataset.embedding model tip": "索引模型可以将自然语言转成向量,用于进行语义检索。\n注意,不同索引模型无法一起使用,选择完索引模型后将无法修改。", "core.dataset.embedding model tip": "索引模型可以将知识库内容转成向量,用于进行语义检索。注意,不同索引模型的知识库无法同时查询,切换索引模型需重建全量向量索引,请慎重选择。",
"core.dataset.error.Data not found": "数据不存在或已被删除", "core.dataset.error.Data not found": "数据不存在或已被删除",
"core.dataset.error.Start Sync Failed": "开始同步失败", "core.dataset.error.Start Sync Failed": "开始同步失败",
"core.dataset.error.unExistDataset": "知识库不存在", "core.dataset.error.unExistDataset": "知识库不存在",
...@@ -521,8 +523,15 @@ ...@@ -521,8 +523,15 @@
"core.dataset.test.Test": "测试", "core.dataset.test.Test": "测试",
"core.dataset.test.Test Result": "测试结果", "core.dataset.test.Test Result": "测试结果",
"core.dataset.test.Test Text": "单个文本测试", "core.dataset.test.Test Text": "单个文本测试",
"core.dataset.test.Test Text Placeholder": "输入需要测试的文本", "core.dataset.test.Test Text Placeholder": "输入需要测试的内容",
"core.dataset.test.image_expired": "图片已过期",
"core.dataset.test.image_search_disabled_tip": "请配置图片理解模型或多模态索引模型",
"core.dataset.test.image_token": "[图片]",
"core.dataset.test.input_title": "输入测试内容",
"core.dataset.test.max_images_tip": "最多支持上传10张图片",
"core.dataset.test.search_config": "搜索配置",
"core.dataset.test.Test params": "测试参数", "core.dataset.test.Test params": "测试参数",
"core.dataset.test.upload_image": "上传图片",
"core.dataset.test.delete test history": "删除该测试结果", "core.dataset.test.delete test history": "删除该测试结果",
"core.dataset.test.test history": "测试历史", "core.dataset.test.test history": "测试历史",
"core.dataset.test.test result placeholder": "测试结果将在这里展示", "core.dataset.test.test result placeholder": "测试结果将在这里展示",
...@@ -727,6 +736,7 @@ ...@@ -727,6 +736,7 @@
"dataset.data.Index Placeholder": "输入索引文本内容", "dataset.data.Index Placeholder": "输入索引文本内容",
"dataset.data.Input Success Tip": "导入数据成功", "dataset.data.Input Success Tip": "导入数据成功",
"dataset.data.Update Success Tip": "更新数据成功", "dataset.data.Update Success Tip": "更新数据成功",
"dataset.data.Update Index Timeout Tip": "更新索引超时,请稍后刷新数据查看结果",
"dataset.data.edit.Index": "数据索引({{amount}})", "dataset.data.edit.Index": "数据索引({{amount}})",
"dataset.dataset_name": "知识库名称", "dataset.dataset_name": "知识库名称",
"dataset.deleteFolderTips": "确认删除该文件夹及其包含的所有知识库?删除后数据无法恢复,请确认!", "dataset.deleteFolderTips": "确认删除该文件夹及其包含的所有知识库?删除后数据无法恢复,请确认!",
......
...@@ -36,6 +36,8 @@ ...@@ -36,6 +36,8 @@
"common.error.unKnow": "未知错误", "common.error.unKnow": "未知错误",
"common_dataset": "通用知识库", "common_dataset": "通用知识库",
"common_dataset_desc": "通过导入文件、网页链接或手动录入形式构建知识库", "common_dataset_desc": "通过导入文件、网页链接或手动录入形式构建知识库",
"create_dataset_title": "创建{{name}}",
"dataset_name_placeholder": "给知识库取一个名字",
"confirm_delete_collection": "确认删除 {{num }} 个文件?", "confirm_delete_collection": "确认删除 {{num }} 个文件?",
"confirm_import_images": "共 {{num}} 张图片 | 确认创建", "confirm_import_images": "共 {{num}} 张图片 | 确认创建",
"confirm_to_rebuild_embedding_tip": "确认为知识库切换索引?\n切换索引是一个非常重量的操作,需要对您知识库内所有数据进行重新索引,时间可能较长,请确保账号内剩余积分充足。\n\n此外,你还需要注意修改选择该知识库的应用,避免它们与其他索引模型知识库混用。", "confirm_to_rebuild_embedding_tip": "确认为知识库切换索引?\n切换索引是一个非常重量的操作,需要对您知识库内所有数据进行重新索引,时间可能较长,请确保账号内剩余积分充足。\n\n此外,你还需要注意修改选择该知识库的应用,避免它们与其他索引模型知识库混用。",
...@@ -86,8 +88,14 @@ ...@@ -86,8 +88,14 @@
"file_model_function_tip": "用于增强索引和 QA 生成", "file_model_function_tip": "用于增强索引和 QA 生成",
"filename": "文件名", "filename": "文件名",
"folder_dataset": "文件夹", "folder_dataset": "文件夹",
"generate_index": "更新索引",
"image_auto_parse": "图片自动索引", "image_auto_parse": "图片自动索引",
"image_auto_parse_tips": "调用 VLM 自动标注文档里的图片,并生成额外的检索索引", "image_auto_parse_tips": "调用 VLM 自动标注文档里的图片,并生成额外的检索索引",
"image_auto_parse_tip_commercial": "请升级商业版后使用该功能",
"image_auto_parse_tip_multimodal_with_vlm": "为文档中的图片生成图片向量索引和文本描述索引,支持以图搜图",
"image_auto_parse_tip_multimodal_without_vlm": "使用多模态模型为图片生成向量索引,支持以图搜图",
"image_auto_parse_tip_vlm_only": "调用 VLM 自动标注文档里的图片,并生成文本描述索引",
"image_auto_parse_tip_no_vlm_or_multimodal": "需配置图片理解模型,或切换多模态向量模型后,方可启用",
"images_creating": "正在创建", "images_creating": "正在创建",
"immediate_sync": "立即同步", "immediate_sync": "立即同步",
"import_confirm": "确认上传", "import_confirm": "确认上传",
...@@ -186,6 +194,7 @@ ...@@ -186,6 +194,7 @@
"uploading_progress": "上传中: {{num}}%", "uploading_progress": "上传中: {{num}}%",
"vector_model_max_tokens_tip": "每个分块数据,最大长度为 3000 tokens", "vector_model_max_tokens_tip": "每个分块数据,最大长度为 3000 tokens",
"vllm_model": "图片理解模型", "vllm_model": "图片理解模型",
"vllm_model_tip": "自动标注文档里的图片并生成文本描述,辅助文本检索",
"website_dataset": "Web 站点同步", "website_dataset": "Web 站点同步",
"website_dataset_desc": "通过爬虫,批量爬取网页数据构建知识库", "website_dataset_desc": "通过爬虫,批量爬取网页数据构建知识库",
"website_info": "网站信息", "website_info": "网站信息",
......
{ {
"Image_Preview": "图片预览", "Image_Preview": "图片预览",
"Image_dataset_requires_VLM_model_to_be_configured": "图片数据集需要配置图片理解模型(VLM)才能使用,请先在模型配置中添加支持图片理解的模型", "Image_dataset_requires_VLM_model_to_be_configured": "图片数据集需要配置图片理解模型或多模态索引模型才能使用,请先在模型配置中添加对应模型",
"click_to_view_raw_source": "点击查看来源", "click_to_view_raw_source": "点击查看来源",
"file_name": "文件名", "file_name": "文件名",
"file_size": "文件大小", "file_size": "文件大小",
......
...@@ -95,6 +95,7 @@ ...@@ -95,6 +95,7 @@
"model.vision": "支援圖片識別", "model.vision": "支援圖片識別",
"model.vision_tag": "視覺", "model.vision_tag": "視覺",
"model.vision_tip": "如果模型支援圖片識別,則開啟該開關。", "model.vision_tip": "如果模型支援圖片識別,則開啟該開關。",
"model.embedding_vision_tip": "開啟後該索引模型可接收圖片輸入,並用於圖片向量索引和圖搜圖。",
"model.voices": "聲音角色", "model.voices": "聲音角色",
"model.voices_tip": "透過一個陣列設定多個,例如:\n[\n {\n \"label\": \"Alloy\",\n \"value\": \"alloy\"\n },\n {\n \"label\": \"Echo\",\n \"value\": \"echo\"\n }\n]", "model.voices_tip": "透過一個陣列設定多個,例如:\n[\n {\n \"label\": \"Alloy\",\n \"value\": \"alloy\"\n },\n {\n \"label\": \"Echo\",\n \"value\": \"echo\"\n }\n]",
"model_provider": "模型提供者", "model_provider": "模型提供者",
......
...@@ -81,7 +81,7 @@ ...@@ -81,7 +81,7 @@
"view_chart": "圖表", "view_chart": "圖表",
"view_table": "表格", "view_table": "表格",
"vlm_model": "圖片理解模型", "vlm_model": "圖片理解模型",
"vlm_model_tip": "用於知識庫中對文件中的圖片進行額外的索引生成", "vlm_model_tip": "自動標註文件裡的圖片並生成文字描述,輔助文字檢索",
"volunme_of_failed_calls": "調用失敗量", "volunme_of_failed_calls": "調用失敗量",
"waiting_test": "等待測試" "waiting_test": "等待測試"
} }
...@@ -237,6 +237,8 @@ ...@@ -237,6 +237,8 @@
"core.ai.Prompt": "提示詞", "core.ai.Prompt": "提示詞",
"core.ai.Support tool": "工具調用", "core.ai.Support tool": "工具調用",
"core.ai.model.Dataset Agent Model": "檔案處理模型", "core.ai.model.Dataset Agent Model": "檔案處理模型",
"core.ai.model.multimodal": "多模態",
"core.ai.model.multimodal_tip": "多模態索引模型可以給圖片生成向量。",
"core.ai.model.Vector Model": "索引模型", "core.ai.model.Vector Model": "索引模型",
"core.ai.model.doc_index_and_dialog": "文件索引與對話索引", "core.ai.model.doc_index_and_dialog": "文件索引與對話索引",
"core.app.Api request": "API 存取", "core.app.Api request": "API 存取",
...@@ -453,7 +455,7 @@ ...@@ -453,7 +455,7 @@
"core.dataset.data.Search data placeholder": "搜尋相關資料", "core.dataset.data.Search data placeholder": "搜尋相關資料",
"core.dataset.data.Updated": "已更新", "core.dataset.data.Updated": "已更新",
"core.dataset.data.group": "組", "core.dataset.data.group": "組",
"core.dataset.embedding model tip": "索引模型可以將自然語言轉換成向量,用於進行語意搜尋。\n注意,不同索引模型無法一起使用。選擇索引模型後就無法修改。", "core.dataset.embedding model tip": "索引模型可以將知識庫內容轉成向量,用於進行語意檢索。注意,不同索引模型的知識庫無法同時查詢,切換索引模型需重建全量向量索引,請慎重選擇。",
"core.dataset.error.Data not found": "資料不存在或已被刪除", "core.dataset.error.Data not found": "資料不存在或已被刪除",
"core.dataset.error.Start Sync Failed": "開始同步失敗", "core.dataset.error.Start Sync Failed": "開始同步失敗",
"core.dataset.error.unExistDataset": "知識庫不存在", "core.dataset.error.unExistDataset": "知識庫不存在",
...@@ -516,8 +518,15 @@ ...@@ -516,8 +518,15 @@
"core.dataset.test.Test": "測試", "core.dataset.test.Test": "測試",
"core.dataset.test.Test Result": "測試結果", "core.dataset.test.Test Result": "測試結果",
"core.dataset.test.Test Text": "單一文字測試", "core.dataset.test.Test Text": "單一文字測試",
"core.dataset.test.Test Text Placeholder": "輸入需要測試的文字", "core.dataset.test.Test Text Placeholder": "輸入需要測試的內容",
"core.dataset.test.image_expired": "圖片已過期",
"core.dataset.test.image_search_disabled_tip": "請配置圖片理解模型或多模態索引模型",
"core.dataset.test.image_token": "[圖片]",
"core.dataset.test.input_title": "輸入測試內容",
"core.dataset.test.max_images_tip": "最多支援上傳10張圖片",
"core.dataset.test.search_config": "搜尋設定",
"core.dataset.test.Test params": "測試參數", "core.dataset.test.Test params": "測試參數",
"core.dataset.test.upload_image": "上傳圖片",
"core.dataset.test.delete test history": "刪除此測試結果", "core.dataset.test.delete test history": "刪除此測試結果",
"core.dataset.test.test history": "測試歷史", "core.dataset.test.test history": "測試歷史",
"core.dataset.test.test result placeholder": "測試結果將顯示在這裡", "core.dataset.test.test result placeholder": "測試結果將顯示在這裡",
...@@ -722,6 +731,7 @@ ...@@ -722,6 +731,7 @@
"dataset.data.Index Placeholder": "輸入索引文字內容", "dataset.data.Index Placeholder": "輸入索引文字內容",
"dataset.data.Input Success Tip": "匯入資料成功", "dataset.data.Input Success Tip": "匯入資料成功",
"dataset.data.Update Success Tip": "更新資料成功", "dataset.data.Update Success Tip": "更新資料成功",
"dataset.data.Update Index Timeout Tip": "更新索引逾時,請稍後重新整理資料查看結果",
"dataset.data.edit.Index": "資料索引({{amount}})", "dataset.data.edit.Index": "資料索引({{amount}})",
"dataset.dataset_name": "知識庫名稱", "dataset.dataset_name": "知識庫名稱",
"dataset.deleteFolderTips": "確認刪除此資料夾及其包含的所有知識庫?刪除後資料無法復原,請確認!", "dataset.deleteFolderTips": "確認刪除此資料夾及其包含的所有知識庫?刪除後資料無法復原,請確認!",
......
...@@ -36,6 +36,8 @@ ...@@ -36,6 +36,8 @@
"common.error.unKnow": "未知錯誤", "common.error.unKnow": "未知錯誤",
"common_dataset": "通用資料集", "common_dataset": "通用資料集",
"common_dataset_desc": "通過導入文件、網頁鏈接或手動錄入形式構建知識庫", "common_dataset_desc": "通過導入文件、網頁鏈接或手動錄入形式構建知識庫",
"create_dataset_title": "建立{{name}}",
"dataset_name_placeholder": "給知識庫取一個名字",
"confirm_delete_collection": "確認刪除 {{num }} 個文件?", "confirm_delete_collection": "確認刪除 {{num }} 個文件?",
"confirm_import_images": "共 {{num}} 張圖片 | 確認創建", "confirm_import_images": "共 {{num}} 張圖片 | 確認創建",
"confirm_to_rebuild_embedding_tip": "確定要為資料集切換索引嗎?\n切換索引是一個重要的操作,需要對您資料集內所有資料重新建立索引,可能需要較長時間,請確保帳號內剩餘點數充足。\n\n此外,您還需要注意修改使用此資料集的應用程式,避免與其他索引模型資料集混用。", "confirm_to_rebuild_embedding_tip": "確定要為資料集切換索引嗎?\n切換索引是一個重要的操作,需要對您資料集內所有資料重新建立索引,可能需要較長時間,請確保帳號內剩餘點數充足。\n\n此外,您還需要注意修改使用此資料集的應用程式,避免與其他索引模型資料集混用。",
...@@ -86,8 +88,14 @@ ...@@ -86,8 +88,14 @@
"file_model_function_tip": "用於增強索引和問答生成", "file_model_function_tip": "用於增強索引和問答生成",
"filename": "檔案名稱", "filename": "檔案名稱",
"folder_dataset": "資料夾", "folder_dataset": "資料夾",
"generate_index": "更新索引",
"image_auto_parse": "圖片自動索引", "image_auto_parse": "圖片自動索引",
"image_auto_parse_tips": "呼叫 VLM 自動標註文件裡的圖片,並生成額外的檢索索引", "image_auto_parse_tips": "呼叫 VLM 自動標註文件裡的圖片,並生成額外的檢索索引",
"image_auto_parse_tip_commercial": "請升級商業版後使用該功能",
"image_auto_parse_tip_multimodal_with_vlm": "為文件中的圖片生成圖片向量索引和文字描述索引,支援以圖搜圖",
"image_auto_parse_tip_multimodal_without_vlm": "使用多模態模型為圖片生成向量索引,支援以圖搜圖",
"image_auto_parse_tip_vlm_only": "呼叫 VLM 自動標註文件裡的圖片,並生成文字描述索引",
"image_auto_parse_tip_no_vlm_or_multimodal": "需設定圖片理解模型,或切換多模態向量模型後,方可啟用",
"images_creating": "正在創建", "images_creating": "正在創建",
"immediate_sync": "立即同步", "immediate_sync": "立即同步",
"import_confirm": "確認上傳", "import_confirm": "確認上傳",
...@@ -186,6 +194,7 @@ ...@@ -186,6 +194,7 @@
"uploading_progress": "上傳中: {{num}}%", "uploading_progress": "上傳中: {{num}}%",
"vector_model_max_tokens_tip": "每個分塊資料,最大長度為 3000 tokens", "vector_model_max_tokens_tip": "每個分塊資料,最大長度為 3000 tokens",
"vllm_model": "圖片理解模型", "vllm_model": "圖片理解模型",
"vllm_model_tip": "自動標註文件裡的圖片並生成文字描述,輔助文字檢索",
"website_dataset": "網站同步", "website_dataset": "網站同步",
"website_dataset_desc": "通過爬蟲,批量爬取網頁數據構建知識庫", "website_dataset_desc": "通過爬蟲,批量爬取網頁數據構建知識庫",
"website_info": "網站資訊", "website_info": "網站資訊",
......
{ {
"Image_Preview": "圖片預覽", "Image_Preview": "圖片預覽",
"Image_dataset_requires_VLM_model_to_be_configured": "圖片數據集需要配置圖片理解模型(VLM)才能使用,請先在模型配置中添加支持圖片理解的模型", "Image_dataset_requires_VLM_model_to_be_configured": "圖片數據集需要設定圖片理解模型或多模態索引模型才能使用,請先在模型設定中新增對應模型",
"click_to_view_raw_source": "點選檢視原始來源", "click_to_view_raw_source": "點選檢視原始來源",
"file_name": "檔案名稱", "file_name": "檔案名稱",
"file_size": "檔案大小", "file_size": "檔案大小",
......
...@@ -8,9 +8,11 @@ import MySelect, { type SelectProps } from '@fastgpt/web/components/common/MySel ...@@ -8,9 +8,11 @@ import MySelect, { type SelectProps } from '@fastgpt/web/components/common/MySel
import MultipleRowSelect from '@fastgpt/web/components/common/MySelect/MultipleRowSelect'; import MultipleRowSelect from '@fastgpt/web/components/common/MySelect/MultipleRowSelect';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import TestModeBetaTag from '@/components/core/ai/TestModeBetaTag'; import TestModeBetaTag from '@/components/core/ai/TestModeBetaTag';
import MultimodalTag from '@/components/core/ai/MultimodelTag';
import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import React, { useCallback, useMemo, useState } from 'react'; import React, { useCallback, useMemo } from 'react';
import { ModelTypeEnum } from '@fastgpt/global/core/ai/constants';
type Props = SelectProps & { type Props = SelectProps & {
disableTip?: string; disableTip?: string;
...@@ -18,14 +20,50 @@ type Props = SelectProps & { ...@@ -18,14 +20,50 @@ type Props = SelectProps & {
cacheModel?: boolean; cacheModel?: boolean;
}; };
const modelAvatarSizeMap = {
sm: '1rem',
md: '1.2rem',
lg: '1.4rem'
} as const;
const getModelAvatarSize = (size?: Props['size']) => {
if (typeof size === 'string' && size in modelAvatarSizeMap) {
return modelAvatarSizeMap[size as keyof typeof modelAvatarSizeMap];
}
return modelAvatarSizeMap.md;
};
const isTestModeModel = (model?: SystemModelItemType) => { const isTestModeModel = (model?: SystemModelItemType) => {
return !!model?.testMode; return !!model?.testMode;
}; };
const isMultimodalEmbeddingModel = (model?: SystemModelItemType) => {
return model?.type === ModelTypeEnum.embedding && !!model.vision;
};
const SelectorActiveModelTags = React.memo(function SelectorActiveModelTags({
model
}: {
model?: SystemModelItemType;
}) {
const showTestModeTip = isTestModeModel(model);
const showMultimodalTip = isMultimodalEmbeddingModel(model);
if (!showTestModeTip && !showMultimodalTip) return null;
const SelectorActiveTestModeTip = React.memo(function SelectorActiveTestModeTip() {
return ( return (
<Box position={'absolute'} top={'50%'} right={'40px'} transform={'translateY(-50%)'} zIndex={3}> <Box
<TestModeBetaTag /> position={'absolute'}
top={'50%'}
right={'40px'}
transform={'translateY(-50%)'}
zIndex={3}
display={'flex'}
alignItems={'center'}
gap={1}
>
{showTestModeTip && <TestModeBetaTag />}
{showMultimodalTip && <MultimodalTag />}
</Box> </Box>
); );
}); });
...@@ -33,14 +71,16 @@ const SelectorActiveTestModeTip = React.memo(function SelectorActiveTestModeTip( ...@@ -33,14 +71,16 @@ const SelectorActiveTestModeTip = React.memo(function SelectorActiveTestModeTip(
const ModelOptionLabel = React.memo(function ModelOptionLabel({ const ModelOptionLabel = React.memo(function ModelOptionLabel({
name, name,
showTestModeTip, showTestModeTip,
showMultimodalTip,
noOfLines noOfLines
}: { }: {
name: string; name: string;
showTestModeTip: boolean; showTestModeTip: boolean;
showMultimodalTip?: boolean;
noOfLines?: ResponsiveValue<number>; noOfLines?: ResponsiveValue<number>;
}) { }) {
return ( return (
<Flex alignItems={'center'} flex={'1 1 0'} minW={0} overflow={'hidden'}> <Flex alignItems={'center'} flex={'1 1 0'} w={'100%'} minW={0} overflow={'hidden'}>
<Box noOfLines={noOfLines ?? 1} flex={'1 1 0'} minW={0} overflow={'hidden'}> <Box noOfLines={noOfLines ?? 1} flex={'1 1 0'} minW={0} overflow={'hidden'}>
{name} {name}
</Box> </Box>
...@@ -49,6 +89,11 @@ const ModelOptionLabel = React.memo(function ModelOptionLabel({ ...@@ -49,6 +89,11 @@ const ModelOptionLabel = React.memo(function ModelOptionLabel({
<TestModeBetaTag /> <TestModeBetaTag />
</Box> </Box>
)} )}
{showMultimodalTip && (
<Box ml={1} flexShrink={0} pointerEvents={'auto'}>
<MultimodalTag />
</Box>
)}
</Flex> </Flex>
); );
}); });
...@@ -86,15 +131,7 @@ const OneRowSelector = ({ ...@@ -86,15 +131,7 @@ const OneRowSelector = ({
} }
); );
const avatarSize = useMemo(() => { const avatarSize = useMemo(() => getModelAvatarSize(props.size), [props.size]);
const size = {
sm: '1rem',
md: '1.2rem',
lg: '1.4rem'
};
//@ts-ignore
return props.size ? size[props.size] : size['md'];
}, [props.size]);
const allModels = useMemo( const allModels = useMemo(
() => [ () => [
...llmModelList, ...llmModelList,
...@@ -123,7 +160,7 @@ const OneRowSelector = ({ ...@@ -123,7 +160,7 @@ const OneRowSelector = ({
return { return {
value: item.value, value: item.value,
label: ( label: (
<Flex alignItems={'center'} py={1} minW={0}> <Flex alignItems={'center'} py={1} w={'100%'} minW={0}>
<Avatar <Avatar
borderRadius={'0'} borderRadius={'0'}
mr={2} mr={2}
...@@ -135,6 +172,7 @@ const OneRowSelector = ({ ...@@ -135,6 +172,7 @@ const OneRowSelector = ({
name={modelData.name} name={modelData.name}
noOfLines={noOfLines} noOfLines={noOfLines}
showTestModeTip={isTestModeModel(modelData)} showTestModeTip={isTestModeModel(modelData)}
showMultimodalTip={isMultimodalEmbeddingModel(modelData)}
/> />
</Flex> </Flex>
) )
...@@ -174,6 +212,7 @@ const OneRowSelector = ({ ...@@ -174,6 +212,7 @@ const OneRowSelector = ({
name={selectedModelData.name} name={selectedModelData.name}
noOfLines={noOfLines} noOfLines={noOfLines}
showTestModeTip={false} showTestModeTip={false}
showMultimodalTip={false}
/> />
</Flex> </Flex>
) : undefined ) : undefined
...@@ -187,7 +226,7 @@ const OneRowSelector = ({ ...@@ -187,7 +226,7 @@ const OneRowSelector = ({
}} }}
/> />
</MyTooltip> </MyTooltip>
{isTestModeModel(selectedModelData) && <SelectorActiveTestModeTip />} <SelectorActiveModelTags model={selectedModelData} />
</Box> </Box>
); );
}; };
...@@ -238,21 +277,15 @@ const MultipleRowSelector = ({ ...@@ -238,21 +277,15 @@ const MultipleRowSelector = ({
myModels myModels
]); ]);
const [value, setValue] = useState<string[]>([]); const avatarSize = useMemo(() => getModelAvatarSize(props.size), [props.size]);
const avatarSize = useMemo(() => {
const size = {
sm: '1rem',
md: '1.2rem',
lg: '1.4rem'
};
//@ts-ignore
return props.size ? size[props.size] : size['md'];
}, [props.size]);
const selectedModelData = useMemo( const selectedModelData = useMemo(
() => modelList.find((model) => model?.model === props.value), () => modelList.find((model) => model?.model === props.value),
[modelList, props.value] [modelList, props.value]
); );
const value = useMemo(
() => (selectedModelData ? [selectedModelData.provider, selectedModelData.model] : []),
[selectedModelData]
);
const selectorList = useMemo(() => { const selectorList = useMemo(() => {
const renderList = getModelProviders(i18n.language).map<{ const renderList = getModelProviders(i18n.language).map<{
...@@ -285,7 +318,13 @@ const MultipleRowSelector = ({ ...@@ -285,7 +318,13 @@ const MultipleRowSelector = ({
provider?.children.push({ provider?.children.push({
label: ( label: (
<ModelOptionLabel name={modelData.name} showTestModeTip={isTestModeModel(modelData)} /> <Flex w={'100%'} minW={0}>
<ModelOptionLabel
name={modelData.name}
showTestModeTip={isTestModeModel(modelData)}
showMultimodalTip={isMultimodalEmbeddingModel(modelData)}
/>
</Flex>
), ),
value: modelData.model value: modelData.model
}); });
...@@ -304,13 +343,9 @@ const MultipleRowSelector = ({ ...@@ -304,13 +343,9 @@ const MultipleRowSelector = ({
const SelectedLabel = useMemo(() => { const SelectedLabel = useMemo(() => {
if (loading) return <>{t('common:model_loading')}</>; if (loading) return <>{t('common:model_loading')}</>;
if (!props.value) return <>{t('common:not_model_config')}</>; if (!props.value) return <>{t('common:not_model_config')}</>;
const modelData = modelList.find((model) => model?.model === props.value); if (!selectedModelData) return <>{t('common:not_model_config')}</>;
if (!modelData) return <>{t('common:not_model_config')}</>;
setValue([modelData.provider, props.value]); const avatar = getModelProvider(selectedModelData.provider)?.avatar;
const avatar = getModelProvider(modelData.provider)?.avatar;
return ( return (
<Flex alignItems={'center'} py={1} minW={0} overflow={'hidden'}> <Flex alignItems={'center'} py={1} minW={0} overflow={'hidden'}>
...@@ -321,10 +356,15 @@ const MultipleRowSelector = ({ ...@@ -321,10 +356,15 @@ const MultipleRowSelector = ({
fallbackSrc={HUGGING_FACE_ICON} fallbackSrc={HUGGING_FACE_ICON}
w={avatarSize} w={avatarSize}
/> />
<ModelOptionLabel name={modelData.name} noOfLines={noOfLines} showTestModeTip={false} /> <ModelOptionLabel
name={selectedModelData.name}
noOfLines={noOfLines}
showTestModeTip={false}
showMultimodalTip={false}
/>
</Flex> </Flex>
); );
}, [loading, props.value, t, modelList, getModelProvider, avatarSize, noOfLines]); }, [loading, props.value, t, selectedModelData, getModelProvider, avatarSize, noOfLines]);
return ( return (
<Box <Box
...@@ -351,7 +391,7 @@ const MultipleRowSelector = ({ ...@@ -351,7 +391,7 @@ const MultipleRowSelector = ({
}} }}
/> />
</MyTooltip> </MyTooltip>
{isTestModeModel(selectedModelData) && <SelectorActiveTestModeTip />} <SelectorActiveModelTags model={selectedModelData} />
</Box> </Box>
); );
}; };
......
import { Box } from '@chakra-ui/react';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useTranslation } from 'next-i18next';
import React from 'react';
const MultimodalTag = React.memo(function MultimodalTag() {
const { t } = useTranslation();
return (
<MyTooltip label={t('common:core.ai.model.multimodal_tip')} shouldWrapChildren={false}>
<Box
display={'inline-flex'}
alignItems={'center'}
justifyContent={'center'}
flexShrink={0}
px={'8px'}
py={'4px'}
borderRadius={'6px'}
bg={'#F0FBFF'}
color={'#005B9C'}
fontSize={'10px'}
lineHeight={'14px'}
fontWeight={500}
whiteSpace={'nowrap'}
>
{t('common:core.ai.model.multimodal')}
</Box>
</MyTooltip>
);
});
export default MultimodalTag;
...@@ -10,7 +10,9 @@ import { type AdminFbkType } from '@fastgpt/global/core/chat/type'; ...@@ -10,7 +10,9 @@ import { type AdminFbkType } from '@fastgpt/global/core/chat/type';
import SelectCollections from '@/web/core/dataset/components/SelectCollections'; import SelectCollections from '@/web/core/dataset/components/SelectCollections';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
const InputDataModal = dynamic(() => import('@/pageComponents/dataset/detail/InputDataModal')); const InputDataModal = dynamic(
() => import('@/pageComponents/dataset/detail/components/InputDataModal')
);
export type AdminMarkType = { export type AdminMarkType = {
feedbackDataId?: string; feedbackDataId?: string;
......
/* eslint-disable react-hooks/static-components */
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Box, Flex, type BoxProps, useDisclosure, HStack, Grid } from '@chakra-ui/react'; import { Box, Flex, type BoxProps, useDisclosure, HStack, Grid } from '@chakra-ui/react';
import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type'; import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type';
...@@ -19,6 +20,7 @@ import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; ...@@ -19,6 +20,7 @@ import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import { completionFinishReasonMap } from '@fastgpt/global/core/ai/constants'; import { completionFinishReasonMap } from '@fastgpt/global/core/ai/constants';
import { useSafeTranslation } from '@fastgpt/web/hooks/useSafeTranslation'; import { useSafeTranslation } from '@fastgpt/web/hooks/useSafeTranslation';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import ImagePreviewToken from '@/components/core/dataset/ImagePreviewToken';
const RequestIdDetailModal = dynamic(() => import('@/components/core/ai/requestId'), { const RequestIdDetailModal = dynamic(() => import('@/components/core/ai/requestId'), {
ssr: false ssr: false
...@@ -35,6 +37,36 @@ type sideTabItemType = { ...@@ -35,6 +37,36 @@ type sideTabItemType = {
children: sideTabItemType[]; children: sideTabItemType[];
}; };
const QueryWithImages = React.memo(function QueryWithImages({
query,
queryImages,
datasetId
}: {
query?: string;
queryImages: NonNullable<ChatHistoryItemResType['queryImages']>;
datasetId?: string;
}) {
return (
<Box
border={'1px solid'}
borderColor={'myGray.200'}
borderRadius={'6px'}
bg={'myGray.50'}
color={'myGray.900'}
minH={'32px'}
px={3}
py={2}
>
{!!query && (
<Box whiteSpace={'pre-wrap'} mb={queryImages.length > 0 ? 2 : 0}>
{query}
</Box>
)}
<ImagePreviewToken images={queryImages} datasetId={datasetId} />
</Box>
);
});
/* Per response value */ /* Per response value */
export const WholeResponseContent = ({ export const WholeResponseContent = ({
activeModule, activeModule,
...@@ -50,6 +82,7 @@ export const WholeResponseContent = ({ ...@@ -50,6 +82,7 @@ export const WholeResponseContent = ({
onOpenRequestIdDetail?: (requestId: string) => void; onOpenRequestIdDetail?: (requestId: string) => void;
}) => { }) => {
const { t } = useSafeTranslation(); const { t } = useSafeTranslation();
const queryPreviewDatasetId = activeModule?.quoteList?.[0]?.datasetId;
// Auto scroll to top // Auto scroll to top
const ContentRef = useRef<HTMLDivElement>(null); const ContentRef = useRef<HTMLDivElement>(null);
...@@ -285,7 +318,20 @@ export const WholeResponseContent = ({ ...@@ -285,7 +318,20 @@ export const WholeResponseContent = ({
)} )}
<Row label={t('chat:step_query')} value={activeModule?.stepQuery} /> <Row label={t('chat:step_query')} value={activeModule?.stepQuery} />
{activeModule?.queryImages && activeModule.queryImages.length > 0 ? (
<Row
label={t('common:core.chat.response.module query')}
rawDom={
<QueryWithImages
query={activeModule?.query}
queryImages={activeModule.queryImages}
datasetId={queryPreviewDatasetId}
/>
}
/>
) : (
<Row label={t('common:core.chat.response.module query')} value={activeModule?.query} /> <Row label={t('common:core.chat.response.module query')} value={activeModule?.query} />
)}
<Row <Row
label={t('common:core.chat.response.context total length')} label={t('common:core.chat.response.context total length')}
value={activeModule?.contextTotalLen} value={activeModule?.contextTotalLen}
...@@ -333,10 +379,7 @@ export const WholeResponseContent = ({ ...@@ -333,10 +379,7 @@ export const WholeResponseContent = ({
label={t('common:core.dataset.search.search mode')} label={t('common:core.dataset.search.search mode')}
rawDom={ rawDom={
<Flex border={'base'} borderRadius={'md'} p={2}> <Flex border={'base'} borderRadius={'md'} p={2}>
<Box> <Box>{t(DatasetSearchModeMap[activeModule.searchMode]?.title as any)}</Box>
{/* @ts-ignore */}
{t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
</Box>
{activeModule.embeddingWeight && ( {activeModule.embeddingWeight && (
<>{`(${t('chat:response_hybrid_weight', { <>{`(${t('chat:response_hybrid_weight', {
emb: activeModule.embeddingWeight, emb: activeModule.embeddingWeight,
...@@ -838,7 +881,7 @@ export const ResponseBox = React.memo(function ResponseBox({ ...@@ -838,7 +881,7 @@ export const ResponseBox = React.memo(function ResponseBox({
/* Format response data to slider data */ /* Format response data to slider data */
function pretreatmentResponse(res: ChatHistoryItemResType[]): sideTabItemType[] { function pretreatmentResponse(res: ChatHistoryItemResType[]): sideTabItemType[] {
return res.map((item) => { return res.map((item) => {
let children: sideTabItemType[] = []; const children: sideTabItemType[] = [];
if (item?.toolDetail) children.push(...pretreatmentResponse(item?.toolDetail)); if (item?.toolDetail) children.push(...pretreatmentResponse(item?.toolDetail));
if (item?.pluginDetail) children.push(...pretreatmentResponse(item?.pluginDetail)); if (item?.pluginDetail) children.push(...pretreatmentResponse(item?.pluginDetail));
......
import React, { useEffect, useState } from 'react';
import { Box, Flex, Portal, type BoxProps, type FlexProps } from '@chakra-ui/react';
import { isDatasetFileObjectKey } from '@fastgpt/global/core/dataset/utils';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useSafeTranslation } from '@fastgpt/web/hooks/useSafeTranslation';
import { postGetSearchTestImagePreviewUrls } from '@/web/core/dataset/api/file';
export type ImagePreviewTokenItemType = {
key?: string;
url?: string;
previewUrl?: string;
name?: string;
};
const getDirectPreviewUrl = (image: ImagePreviewTokenItemType) => {
const url = image.previewUrl || image.url || '';
return url && !isDatasetFileObjectKey(url) ? url : '';
};
const ImagePreview = React.memo(function ImagePreview({
image,
datasetId
}: {
image: ImagePreviewTokenItemType;
datasetId?: string;
}) {
const { t } = useSafeTranslation();
const [previewUrl, setPreviewUrl] = useState(() => getDirectPreviewUrl(image));
const [loadFailed, setLoadFailed] = useState(false);
const [hasRefreshed, setHasRefreshed] = useState(false);
useEffect(() => {
setPreviewUrl(getDirectPreviewUrl(image));
setLoadFailed(false);
setHasRefreshed(false);
}, [image]);
useEffect(() => {
if (!image.key || !datasetId || hasRefreshed || (previewUrl && !loadFailed)) return;
let canceled = false;
setHasRefreshed(true);
postGetSearchTestImagePreviewUrls({
datasetId,
keys: [image.key]
})
.then((res) => {
const nextPreviewUrl = res.find((item) => item.key === image.key)?.previewUrl;
if (!canceled && nextPreviewUrl) {
setPreviewUrl(nextPreviewUrl);
setLoadFailed(false);
}
})
.catch(() => {});
return () => {
canceled = true;
};
}, [datasetId, hasRefreshed, image.key, loadFailed, previewUrl]);
if (previewUrl && !loadFailed) {
return (
<Box
as={'img'}
src={previewUrl}
alt={image.name || ''}
w={'80px'}
h={'80px'}
objectFit={'cover'}
borderRadius={'sm'}
onError={() => setLoadFailed(true)}
/>
);
}
return (
<Flex
w={'80px'}
h={'80px'}
flexDir={'column'}
alignItems={'center'}
justifyContent={'center'}
gap={1}
bg={'myGray.50'}
border={'1px dashed'}
borderColor={'myGray.300'}
borderRadius={'sm'}
color={'myGray.500'}
fontSize={'xs'}
lineHeight={'16px'}
>
<MyIcon name={'image'} w={'20px'} h={'20px'} color={'myGray.400'} />
<Box>{t('common:core.dataset.test.image_expired')}</Box>
</Flex>
);
});
const defaultTokenStyles: BoxProps = {
as: 'span',
display: 'inline-flex',
px: 2,
py: 1,
border: '1px solid',
borderColor: 'myGray.200',
borderRadius: 'md',
bg: 'white',
color: 'myGray.700',
cursor: 'default',
lineHeight: '16px',
verticalAlign: 'baseline'
};
const ImagePreviewToken = React.memo(function ImagePreviewToken({
images,
datasetId,
containerProps,
tokenProps
}: {
images: ImagePreviewTokenItemType[];
datasetId?: string;
containerProps?: FlexProps;
tokenProps?: BoxProps;
}) {
const { t } = useSafeTranslation();
const [hoveredImage, setHoveredImage] = useState<
| {
image: ImagePreviewTokenItemType;
top: number;
left: number;
}
| undefined
>();
if (images.length === 0) return null;
return (
<>
<Flex flexWrap={'wrap'} gap={2} {...containerProps}>
{images.map((image, index) => (
<Box
key={`${image.key || image.url || image.previewUrl || index}`}
{...defaultTokenStyles}
{...tokenProps}
onMouseEnter={(e) => {
const rect = e.currentTarget.getBoundingClientRect();
setHoveredImage({
image,
top: rect.bottom + 8,
left: rect.left
});
}}
onMouseLeave={() => setHoveredImage(undefined)}
>
{t('common:core.dataset.test.image_token')}
</Box>
))}
</Flex>
{!!hoveredImage && (
<Portal>
<Flex
position={'fixed'}
zIndex={'tooltip'}
top={`${hoveredImage.top}px`}
left={`${hoveredImage.left}px`}
p={3}
bg={'white'}
borderWidth={'1px'}
borderColor={'borderColor.base'}
borderRadius={'md'}
boxShadow={'2'}
pointerEvents={'none'}
>
<ImagePreview image={hoveredImage.image} datasetId={datasetId} />
</Flex>
</Portal>
)}
</>
);
});
export default ImagePreviewToken;
...@@ -12,9 +12,15 @@ import { SearchScoreTypeEnum, SearchScoreTypeMap } from '@fastgpt/global/core/da ...@@ -12,9 +12,15 @@ import { SearchScoreTypeEnum, SearchScoreTypeMap } from '@fastgpt/global/core/da
import type { ReadCollectionSourceBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import type { ReadCollectionSourceBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api';
import Markdown from '@/components/Markdown'; import Markdown from '@/components/Markdown';
const InputDataModal = dynamic(() => import('@/pageComponents/dataset/detail/InputDataModal')); const InputDataModal = dynamic(
() => import('@/pageComponents/dataset/detail/components/InputDataModal')
);
export type ScoreItemType = SearchDataResponseItemType['score'][0]; export type ScoreItemType = {
type: SearchScoreTypeEnum;
value: number;
index: number;
};
export const scoreTheme: Record< export const scoreTheme: Record<
string, string,
{ {
...@@ -52,32 +58,19 @@ export const formatScore = (score: ScoreItemType[]) => { ...@@ -52,32 +58,19 @@ export const formatScore = (score: ScoreItemType[]) => {
}; };
} }
// rrf -> rerank -> embedding -> fullText 优先级 const scoreList = [
let rrfScore: ScoreItemType | undefined = undefined; SearchScoreTypeEnum.rrf,
let reRankScore: ScoreItemType | undefined = undefined; SearchScoreTypeEnum.reRank,
let embeddingScore: ScoreItemType | undefined = undefined; SearchScoreTypeEnum.embedding,
let fullTextScore: ScoreItemType | undefined = undefined; SearchScoreTypeEnum.fullText
]
score.forEach((item) => { .map((type) => score.find((item) => item.type === type))
if (item.type === SearchScoreTypeEnum.rrf) { .filter((item): item is ScoreItemType => !!item);
rrfScore = item;
} else if (item.type === SearchScoreTypeEnum.reRank) {
reRankScore = item;
} else if (item.type === SearchScoreTypeEnum.embedding) {
embeddingScore = item;
} else if (item.type === SearchScoreTypeEnum.fullText) {
fullTextScore = item;
}
});
const primaryScore = (rrfScore || const primaryScore = scoreList[0];
reRankScore || const secondaryScore = primaryScore
embeddingScore || ? scoreList.slice(1).filter((item) => item.type !== primaryScore.type)
fullTextScore) as unknown as ScoreItemType; : [];
const secondaryScore = [rrfScore, reRankScore, embeddingScore, fullTextScore].filter(
// @ts-ignore
(item) => item && primaryScore && item.type !== primaryScore.type
) as unknown as ScoreItemType[];
return { return {
primaryScore, primaryScore,
...@@ -142,7 +135,10 @@ const QuoteItem = ({ ...@@ -142,7 +135,10 @@ const QuoteItem = ({
</Flex> </Flex>
</MyTooltip> </MyTooltip>
)} )}
{score.secondaryScore.map((item, i) => ( {score.secondaryScore.map((item, i) => {
const theme = scoreTheme[i];
return (
<MyTooltip key={item.type} label={t(SearchScoreTypeMap[item.type]?.desc as any)}> <MyTooltip key={item.type} label={t(SearchScoreTypeMap[item.type]?.desc as any)}>
<Box fontSize={'xs'}> <Box fontSize={'xs'}>
<Flex alignItems={'flex-start'} lineHeight={1.2} mb={1}> <Flex alignItems={'flex-start'} lineHeight={1.2} mb={1}>
...@@ -151,7 +147,11 @@ const QuoteItem = ({ ...@@ -151,7 +147,11 @@ const QuoteItem = ({
borderWidth={'1px'} borderWidth={'1px'}
borderRadius={'sm'} borderRadius={'sm'}
mr={'2px'} mr={'2px'}
{...(scoreTheme[i] && scoreTheme[i])} {...(theme && {
color: theme.color,
bg: theme.bg,
borderColor: theme.borderColor
})}
> >
<Box transform={'scale(0.9)'}>#{item.index + 1}</Box> <Box transform={'scale(0.9)'}>#{item.index + 1}</Box>
</Box> </Box>
...@@ -167,8 +167,8 @@ const QuoteItem = ({ ...@@ -167,8 +167,8 @@ const QuoteItem = ({
w={'100%'} w={'100%'}
size="sm" size="sm"
borderRadius={'20px'} borderRadius={'20px'}
{...(scoreTheme[i] && { {...(theme && {
colorScheme: scoreTheme[i].colorScheme colorScheme: theme.colorScheme
})} })}
bg="#E8EBF0" bg="#E8EBF0"
/> />
...@@ -176,7 +176,8 @@ const QuoteItem = ({ ...@@ -176,7 +176,8 @@ const QuoteItem = ({
</Box> </Box>
</Box> </Box>
</MyTooltip> </MyTooltip>
))} );
})}
</Flex> </Flex>
<Box flex={'1 0 0'}> <Box flex={'1 0 0'}>
......
...@@ -5,7 +5,7 @@ import { ...@@ -5,7 +5,7 @@ import {
DatasetSearchModeMap DatasetSearchModeMap
} from '@fastgpt/global/core/dataset/constants'; } from '@fastgpt/global/core/dataset/constants';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import React, { useEffect, useMemo } from 'react'; import React, { useMemo } from 'react';
import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIcon from '@fastgpt/web/components/common/Icon';
import { getWebLLMModel } from '@/web/common/system/utils'; import { getWebLLMModel } from '@/web/common/system/utils';
...@@ -45,6 +45,15 @@ const SearchParamsTip = ({ ...@@ -45,6 +45,15 @@ const SearchParamsTip = ({
borderWidth={'1px'} borderWidth={'1px'}
borderColor={'primary.1'} borderColor={'primary.1'}
sx={{ sx={{
'& thead > tr': {
borderBottom: 'none !important'
},
'& thead > tr > th': {
borderBottom: 'none !important'
},
'& tbody > tr:first-of-type > td, & tbody > tr:first-of-type > th': {
borderTop: 'none !important'
},
'&::-webkit-scrollbar': { '&::-webkit-scrollbar': {
height: '6px', height: '6px',
borderRadius: '4px' borderRadius: '4px'
......
...@@ -838,12 +838,11 @@ export const ModelEditModal = ({ ...@@ -838,12 +838,11 @@ export const ModelEditModal = ({
data.priceTiers = priceTiers as any; data.priceTiers = priceTiers as any;
} }
for (const key in data) { const modelData = data as Record<string, unknown>;
// @ts-ignore for (const key of Object.keys(modelData)) {
const val = data[key]; const val = modelData[key];
if (val === null || val === undefined || Number.isNaN(val)) { if (val === null || val === undefined || Number.isNaN(val)) {
// @ts-ignore modelData[key] = '';
data[key] = '';
} }
} }
...@@ -915,7 +914,7 @@ export const ModelEditModal = ({ ...@@ -915,7 +914,7 @@ export const ModelEditModal = ({
headerPx={'32px'} headerPx={'32px'}
> >
<ModalBody px={'32px'} py={0}> <ModalBody px={'32px'} py={0}>
<Section title={t('account:model.basic_config_section')}> <Section key={key} title={t('account:model.basic_config_section')}>
<Flex direction={['column', 'row']} gap={[6, 8]} alignItems={['stretch', 'flex-start']}> <Flex direction={['column', 'row']} gap={[6, 8]} alignItems={['stretch', 'flex-start']}>
<Grid flex={'1 0 0'} templateColumns={['1fr', 'repeat(2, minmax(0, 1fr))']} gap={4}> <Grid flex={'1 0 0'} templateColumns={['1fr', 'repeat(2, minmax(0, 1fr))']} gap={4}>
<Field label={t('account:model.model_id')} tip={t('account:model.model_id_tip')}> <Field label={t('account:model.model_id')} tip={t('account:model.model_id_tip')}>
...@@ -1101,6 +1100,19 @@ export const ModelEditModal = ({ ...@@ -1101,6 +1100,19 @@ export const ModelEditModal = ({
</Section> </Section>
)} )}
{isEmbeddingModel && (
<Section title={t('account:model.feature_config_section')}>
<Grid templateColumns={['1fr', 'repeat(2, minmax(0, 1fr))']} gap={4}>
<SwitchField
label={t('account:model.vision')}
tip={t('account:model.embedding_vision_tip')}
field={'vision'}
register={register}
/>
</Grid>
</Section>
)}
{priceUnit && feConfigs?.isPlus && ( {priceUnit && feConfigs?.isPlus && (
<Section title={t('account:model.price_config_section')}> <Section title={t('account:model.price_config_section')}>
{isLLMModel ? ( {isLLMModel ? (
......
...@@ -11,13 +11,12 @@ import { ...@@ -11,13 +11,12 @@ import {
Tr, Tr,
Switch, Switch,
ModalBody, ModalBody,
Input,
ModalFooter, ModalFooter,
Button, Button,
useDisclosure useDisclosure
} from '@chakra-ui/react'; } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import React, { useCallback, useMemo, useRef, useState } from 'react'; import React, { useCallback, useMemo, useState } from 'react';
import MySelect from '@fastgpt/web/components/common/MySelect'; import MySelect from '@fastgpt/web/components/common/MySelect';
import { modelTypeList, ModelTypeEnum } from '@fastgpt/global/core/ai/constants'; import { modelTypeList, ModelTypeEnum } from '@fastgpt/global/core/ai/constants';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput'; import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
...@@ -63,7 +62,8 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -63,7 +62,8 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
const isRoot = userInfo?.username === 'root'; const isRoot = userInfo?.username === 'root';
const [provider, setProvider] = useState<string | ''>(''); const [provider, setProvider] = useState<string | ''>('');
const providerList = useRef<{ label: React.ReactNode; value: string | '' }[]>([ const providerList = useMemo<{ label: React.ReactNode; value: string | '' }[]>(
() => [
{ label: t('common:All'), value: '' }, { label: t('common:All'), value: '' },
...getModelProviders(i18n.language).map((item) => ({ ...getModelProviders(i18n.language).map((item) => ({
label: ( label: (
...@@ -74,13 +74,18 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -74,13 +74,18 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
), ),
value: item.id value: item.id
})) }))
]); ],
[getModelProviders, i18n.language, t]
);
const [modelType, setModelType] = useState<ModelTypeEnum | ''>(''); const [modelType, setModelType] = useState<ModelTypeEnum | ''>('');
const selectModelTypeList = useRef<{ label: string; value: ModelTypeEnum | '' }[]>([ const selectModelTypeList = useMemo<{ label: string; value: ModelTypeEnum | '' }[]>(
() => [
{ label: t('common:All'), value: '' }, { label: t('common:All'), value: '' },
...modelTypeList.map((item) => ({ label: t(item.label), value: item.value })) ...modelTypeList.map((item) => ({ label: t(item.label), value: item.value }))
]); ],
[t]
);
const [search, setSearch] = useState(''); const [search, setSearch] = useState('');
const [showActive, setShowActive] = useState(false); const [showActive, setShowActive] = useState(false);
...@@ -233,10 +238,8 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -233,10 +238,8 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
const filterProviderList = useMemo(() => { const filterProviderList = useMemo(() => {
const allProviderIds: string[] = systemModelList.map((model) => model.provider); const allProviderIds: string[] = systemModelList.map((model) => model.provider);
return providerList.current.filter( return providerList.filter((item) => allProviderIds.includes(item.value) || item.value === '');
(item) => allProviderIds.includes(item.value) || item.value === '' }, [providerList, systemModelList]);
);
}, [systemModelList]);
const { runAsync: onTestModel, loading: testingModel } = useRequest(getTestModel, { const { runAsync: onTestModel, loading: testingModel } = useRequest(getTestModel, {
manual: true, manual: true,
...@@ -263,7 +266,7 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -263,7 +266,7 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
const onCreateModel = (type: ModelTypeEnum) => { const onCreateModel = (type: ModelTypeEnum) => {
const defaultModel = defaultModels[type]; const defaultModel = defaultModels[type];
setEditModelData({ const modelData = {
...defaultModel, ...defaultModel,
model: '', model: '',
name: '', name: '',
...@@ -278,9 +281,10 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -278,9 +281,10 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
isDefault: false, isDefault: false,
isDefaultDatasetTextModel: false, isDefaultDatasetTextModel: false,
isDefaultDatasetImageModel: false, isDefaultDatasetImageModel: false,
// @ts-ignore
type type
}); } as SystemModelItemType;
setEditModelData(modelData);
}; };
const { const {
...@@ -337,7 +341,7 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -337,7 +341,7 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
bg={'myGray.50'} bg={'myGray.50'}
value={modelType} value={modelType}
onChange={setModelType} onChange={setModelType}
list={selectModelTypeList.current} list={selectModelTypeList}
/> />
</HStack> </HStack>
<Box flex={1} /> <Box flex={1} />
...@@ -381,7 +385,7 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -381,7 +385,7 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
</Tr> </Tr>
</Thead> </Thead>
<Tbody> <Tbody>
{modelList.map((item, index) => ( {modelList.map((item) => (
<Tr key={item.model} _hover={{ bg: 'myGray.50' }}> <Tr key={item.model} _hover={{ bg: 'myGray.50' }}>
<Td fontSize={'sm'}> <Td fontSize={'sm'}>
<HStack> <HStack>
...@@ -405,7 +409,9 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -405,7 +409,9 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
)} )}
{item.vision && ( {item.vision && (
<MyTag type="borderFill" colorSchema="green" py={0.5}> <MyTag type="borderFill" colorSchema="green" py={0.5}>
{t('account:model.vision_tag')} {item.type === ModelTypeEnum.llm
? t('account:model.vision_tag')
: t('common:core.ai.model.multimodal')}
</MyTag> </MyTag>
)} )}
{item.toolChoice && ( {item.toolChoice && (
......
...@@ -5,7 +5,7 @@ import { type Dispatch, type MutableRefObject, type SetStateAction, useState } f ...@@ -5,7 +5,7 @@ import { type Dispatch, type MutableRefObject, type SetStateAction, useState } f
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIcon from '@fastgpt/web/components/common/Icon';
import { useCopyData } from '@fastgpt/web/hooks/useCopyData'; import { useCopyData } from '@fastgpt/web/hooks/useCopyData';
import InputDataModal from '@/pageComponents/dataset/detail/InputDataModal'; import InputDataModal from '@/pageComponents/dataset/detail/components/InputDataModal';
const CollectionQuoteItem = ({ const CollectionQuoteItem = ({
quoteRefs, quoteRefs,
......
...@@ -17,24 +17,32 @@ import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; ...@@ -17,24 +17,32 @@ import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyModal from '@fastgpt/web/components/common/MyModal'; import MyModal from '@fastgpt/web/components/common/MyModal';
import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIcon from '@fastgpt/web/components/common/Icon';
import { FolderIcon } from '@fastgpt/global/common/file/image/constants'; import { FolderIcon } from '@fastgpt/global/common/file/image/constants';
import type { ApiDatasetServerType } from '@fastgpt/global/core/dataset/apiDataset/type';
const ApiDatasetForm = ({ const ApiDatasetForm = ({
type, type,
datasetId, datasetId,
form form,
controlWidth
}: { }: {
type: `${DatasetTypeEnum}`; type: `${DatasetTypeEnum}`;
datasetId?: string; datasetId?: string;
form: UseFormReturn< controlWidth?: string | string[];
{ form: UseFormReturn<any>;
apiDatasetServer?: ApiDatasetServerType;
},
any
>;
}) => { }) => {
const { t } = useTranslation(); const { t } = useTranslation();
const { register, setValue, watch } = form; const { register, setValue, watch } = form;
const inputLayoutProps = controlWidth
? {
w: controlWidth,
flex: '0 0 auto' as const
}
: {};
const rowLayoutProps = controlWidth
? {
justifyContent: 'space-between' as const
}
: {};
const rowTopMargin = controlWidth ? 4 : 6;
const apiDatasetServer = watch('apiDatasetServer'); const apiDatasetServer = watch('apiDatasetServer');
const yuqueServer = apiDatasetServer?.yuqueServer; const yuqueServer = apiDatasetServer?.yuqueServer;
...@@ -114,11 +122,9 @@ const ApiDatasetForm = ({ ...@@ -114,11 +122,9 @@ const ApiDatasetForm = ({
closeBaseurlSelectModal(); closeBaseurlSelectModal();
}; };
const renderBaseUrlSelector = () => ( const renderBaseUrlSelector = () => {
<Flex mt={6} alignItems={'center'}> const selectorContent = (
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'}> <>
Base URL
</FormLabel>
<MyBox py={1} fontSize={'sm'} flex={'1 0 0'} overflow="auto" isLoading={isFetching}> <MyBox py={1} fontSize={'sm'} flex={'1 0 0'} overflow="auto" isLoading={isFetching}>
{pathNames} {pathNames}
</MyBox> </MyBox>
...@@ -131,8 +137,24 @@ const ApiDatasetForm = ({ ...@@ -131,8 +137,24 @@ const ApiDatasetForm = ({
> >
{t('dataset:selectDirectory')} {t('dataset:selectDirectory')}
</Button> </Button>
</>
);
return (
<Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'}>
Base URL
</FormLabel>
{controlWidth ? (
<Flex alignItems={'center'} w={controlWidth}>
{selectorContent}
</Flex>
) : (
selectorContent
)}
</Flex> </Flex>
); );
};
// Render the directory selection modal // Render the directory selection modal
const renderDirectoryModal = () => const renderDirectoryModal = () =>
...@@ -156,22 +178,24 @@ const ApiDatasetForm = ({ ...@@ -156,22 +178,24 @@ const ApiDatasetForm = ({
<> <>
{type === DatasetTypeEnum.apiDataset && ( {type === DatasetTypeEnum.apiDataset && (
<> <>
<Flex mt={6} alignItems={'center'}> <Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required> <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
{t('dataset:api_url')} {t('dataset:api_url')}
</FormLabel> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={t('dataset:api_url')} placeholder={t('dataset:api_url')}
maxLength={200} maxLength={200}
{...register('apiDatasetServer.apiServer.baseUrl', { required: true })} {...register('apiDatasetServer.apiServer.baseUrl', { required: true })}
/> />
</Flex> </Flex>
<Flex mt={6} alignItems={'center'}> <Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required> <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
Authorization Authorization
</FormLabel> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={t('dataset:request_headers')} placeholder={t('dataset:request_headers')}
maxLength={2000} maxLength={2000}
...@@ -184,51 +208,36 @@ const ApiDatasetForm = ({ ...@@ -184,51 +208,36 @@ const ApiDatasetForm = ({
)} )}
{type === DatasetTypeEnum.feishu && ( {type === DatasetTypeEnum.feishu && (
<> <>
<Flex mt={6}> <Flex mt={rowTopMargin} {...rowLayoutProps}>
<Flex <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
App ID App ID
</Flex> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={'App ID'} placeholder={'App ID'}
maxLength={200} maxLength={200}
{...register('apiDatasetServer.feishuServer.appId', { required: true })} {...register('apiDatasetServer.feishuServer.appId', { required: true })}
/> />
</Flex> </Flex>
<Flex mt={6}> <Flex mt={rowTopMargin} {...rowLayoutProps}>
<Flex <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
App Secret App Secret
</Flex> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={'App Secret'} placeholder={'App Secret'}
maxLength={200} maxLength={200}
{...register('apiDatasetServer.feishuServer.appSecret', { required: true })} {...register('apiDatasetServer.feishuServer.appSecret', { required: true })}
/> />
</Flex> </Flex>
<Flex mt={6}> <Flex mt={rowTopMargin} {...rowLayoutProps}>
<Flex <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
Folder Token Folder Token
</Flex> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={'Folder Token'} placeholder={'Folder Token'}
maxLength={200} maxLength={200}
...@@ -241,22 +250,24 @@ const ApiDatasetForm = ({ ...@@ -241,22 +250,24 @@ const ApiDatasetForm = ({
)} )}
{type === DatasetTypeEnum.yuque && ( {type === DatasetTypeEnum.yuque && (
<> <>
<Flex mt={6} alignItems={'center'}> <Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required> <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
User ID User ID
</FormLabel> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={'User ID'} placeholder={'User ID'}
maxLength={200} maxLength={200}
{...register('apiDatasetServer.yuqueServer.userId', { required: true })} {...register('apiDatasetServer.yuqueServer.userId', { required: true })}
/> />
</Flex> </Flex>
<Flex mt={6} alignItems={'center'}> <Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required> <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
Token Token
</FormLabel> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={'Token'} placeholder={'Token'}
maxLength={200} maxLength={200}
...@@ -269,33 +280,36 @@ const ApiDatasetForm = ({ ...@@ -269,33 +280,36 @@ const ApiDatasetForm = ({
)} )}
{type === DatasetTypeEnum.dingtalk && ( {type === DatasetTypeEnum.dingtalk && (
<> <>
<Flex mt={6} alignItems={'center'}> <Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required> <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
App Key App Key
</FormLabel> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={'App Key'} placeholder={'App Key'}
maxLength={200} maxLength={200}
{...register('apiDatasetServer.dingtalkServer.appKey', { required: true })} {...register('apiDatasetServer.dingtalkServer.appKey', { required: true })}
/> />
</Flex> </Flex>
<Flex mt={6} alignItems={'center'}> <Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required> <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
App Secret App Secret
</FormLabel> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={'App Secret'} placeholder={'App Secret'}
maxLength={200} maxLength={200}
{...register('apiDatasetServer.dingtalkServer.appSecret', { required: true })} {...register('apiDatasetServer.dingtalkServer.appSecret', { required: true })}
/> />
</Flex> </Flex>
<Flex mt={6} alignItems={'center'}> <Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required> <FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
User ID User ID
</FormLabel> </FormLabel>
<Input <Input
{...inputLayoutProps}
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={'User ID'} placeholder={'User ID'}
maxLength={200} maxLength={200}
......
import React, { useState, useMemo } from 'react'; import React, { useState, useMemo } from 'react';
import { Box, Card, IconButton, Flex, Button, useTheme, Image } from '@chakra-ui/react'; import { Box, Card, IconButton, Flex, Button } from '@chakra-ui/react';
import { getDatasetCollectionById } from '@/web/core/dataset/api/collection'; import { getDatasetCollectionById } from '@/web/core/dataset/api/collection';
import { getDatasetDataList, delOneDatasetDataById } from '@/web/core/dataset/api/data'; import { getDatasetDataList, delOneDatasetDataById } from '@/web/core/dataset/api/data';
import { useToast } from '@fastgpt/web/hooks/useToast'; import { useToast } from '@fastgpt/web/hooks/useToast';
...@@ -8,7 +8,7 @@ import { useTranslation } from 'next-i18next'; ...@@ -8,7 +8,7 @@ import { useTranslation } from 'next-i18next';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIcon from '@fastgpt/web/components/common/Icon';
import MyInput from '@/components/MyInput'; import MyInput from '@/components/MyInput';
import InputDataModal from './InputDataModal'; import InputDataModal from './components/InputDataModal';
import RawSourceBox from '@/components/core/dataset/RawSourceBox'; import RawSourceBox from '@/components/core/dataset/RawSourceBox';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils'; import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
...@@ -175,7 +175,8 @@ const DataCard = () => { ...@@ -175,7 +175,8 @@ const DataCard = () => {
isDisabled={!collection} isDisabled={!collection}
isLoading={isExportChunksLoading} isLoading={isExportChunksLoading}
onClick={() => { onClick={() => {
onExportAllChunks(collection?._id!); if (!collection?._id) return;
onExportAllChunks(collection._id);
}} }}
> >
{t('dataset:collection.export_all_chunks')} {t('dataset:collection.export_all_chunks')}
......
...@@ -135,6 +135,45 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm ...@@ -135,6 +135,45 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
const imageIndex = watch('imageIndex'); const imageIndex = watch('imageIndex');
const indexPrefixTitle = watch('indexPrefixTitle'); const indexPrefixTitle = watch('indexPrefixTitle');
const paragraphChunkAIMode = watch('paragraphChunkAIMode'); const paragraphChunkAIMode = watch('paragraphChunkAIMode');
const imageIndexConfigState = useMemo(() => {
if (!feConfigs?.isPlus) {
return {
disabled: true,
tooltip: t('common:commercial_function_tip'),
tip: t('dataset:image_auto_parse_tip_commercial')
};
}
if (datasetDetail.vectorModel?.vision && datasetDetail.vlmModel) {
return {
disabled: false,
tooltip: '',
tip: t('dataset:image_auto_parse_tip_multimodal_with_vlm')
};
}
if (datasetDetail.vectorModel?.vision) {
return {
disabled: false,
tooltip: '',
tip: t('dataset:image_auto_parse_tip_multimodal_without_vlm')
};
}
if (datasetDetail.vlmModel) {
return {
disabled: false,
tooltip: '',
tip: t('dataset:image_auto_parse_tip_vlm_only')
};
}
return {
disabled: true,
tooltip: t('dataset:image_auto_parse_tip_no_vlm_or_multimodal'),
tip: t('dataset:image_auto_parse_tip_no_vlm_or_multimodal')
};
}, [datasetDetail.vectorModel?.vision, datasetDetail.vlmModel, feConfigs?.isPlus, t]);
const trainingModeList = useMemo(() => { const trainingModeList = useMemo(() => {
const list = { const list = {
...@@ -224,6 +263,12 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm ...@@ -224,6 +263,12 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
} }
}, [trainingType, setValue]); }, [trainingType, setValue]);
useEffect(() => {
if (imageIndexConfigState.disabled && imageIndex) {
setValue('imageIndex', false);
}
}, [imageIndex, imageIndexConfigState.disabled, setValue]);
return ( return (
<> <>
<Box> <Box>
...@@ -310,24 +355,16 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm ...@@ -310,24 +355,16 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
<QuestionTip label={t('dataset:auto_indexes_tips')} /> <QuestionTip label={t('dataset:auto_indexes_tips')} />
</HStack> </HStack>
<HStack flex={'1'} spacing={1}> <HStack flex={'1'} spacing={1}>
<MyTooltip <MyTooltip label={imageIndexConfigState.tooltip}>
label={
!feConfigs?.isPlus
? t('common:commercial_function_tip')
: !datasetDetail?.vlmModel
? t('common:error_vlm_not_config')
: ''
}
>
<Checkbox <Checkbox
isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel} isDisabled={imageIndexConfigState.disabled}
isChecked={imageIndex} isChecked={imageIndex}
{...register('imageIndex')} {...register('imageIndex')}
> >
<FormLabel>{t('dataset:image_auto_parse')}</FormLabel> <FormLabel>{t('dataset:image_auto_parse')}</FormLabel>
</Checkbox> </Checkbox>
</MyTooltip> </MyTooltip>
<QuestionTip label={t('dataset:image_auto_parse_tips')} /> <QuestionTip label={imageIndexConfigState.tip} />
</HStack> </HStack>
</> </>
)} )}
......
...@@ -41,7 +41,7 @@ const EditAPIDatasetInfoModal = ({ ...@@ -41,7 +41,7 @@ const EditAPIDatasetInfoModal = ({
const { runAsync: onSave, loading } = useRequest( const { runAsync: onSave, loading } = useRequest(
(data: EditAPIDatasetInfoFormType) => onEdit(data), (data: EditAPIDatasetInfoFormType) => onEdit(data),
{ {
onSuccess: (res) => { onSuccess: () => {
toast({ toast({
title: t('common:update_success'), title: t('common:update_success'),
status: 'success' status: 'success'
...@@ -72,7 +72,6 @@ const EditAPIDatasetInfoModal = ({ ...@@ -72,7 +72,6 @@ const EditAPIDatasetInfoModal = ({
</Flex> </Flex>
</Flex> </Flex>
)} )}
{/* @ts-ignore */}
<ApiDatasetForm datasetId={datasetDetail._id} type={type} form={form} /> <ApiDatasetForm datasetId={datasetDetail._id} type={type} form={form} />
</ModalBody> </ModalBody>
<ModalFooter> <ModalFooter>
......
...@@ -176,7 +176,7 @@ const Info = ({ datasetId }: { datasetId: string }) => { ...@@ -176,7 +176,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
</Box> </Box>
</MyTooltip> </MyTooltip>
</Flex> </Flex>
<Box pt={2}> <Box pt={2} minW={0} maxW={'100%'} overflow={'hidden'}>
<AIModelSelector <AIModelSelector
w={'100%'} w={'100%'}
value={vectorModel.model} value={vectorModel.model}
...@@ -210,7 +210,7 @@ const Info = ({ datasetId }: { datasetId: string }) => { ...@@ -210,7 +210,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
<FormLabel fontSize={'mini'} fontWeight={'500'}> <FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('common:core.ai.model.Dataset Agent Model')} {t('common:core.ai.model.Dataset Agent Model')}
</FormLabel> </FormLabel>
<Box pt={2}> <Box pt={2} minW={0} maxW={'100%'} overflow={'hidden'}>
<AIModelSelector <AIModelSelector
w={'100%'} w={'100%'}
value={agentModel.model} value={agentModel.model}
...@@ -233,7 +233,7 @@ const Info = ({ datasetId }: { datasetId: string }) => { ...@@ -233,7 +233,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
<FormLabel fontSize={'mini'} fontWeight={'500'}> <FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:vllm_model')} {t('dataset:vllm_model')}
</FormLabel> </FormLabel>
<Box pt={2}> <Box pt={2} minW={0} maxW={'100%'} overflow={'hidden'}>
<AIModelSelector <AIModelSelector
w={'100%'} w={'100%'}
value={vlmModel?.model} value={vlmModel?.model}
......
import React from 'react';
import { Box, CircularProgress, Flex } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { searchTestImageThumbProps } from '../constants';
import type { SearchTestImageRef } from '../type';
type SearchTestImagePreviewListItem =
| (SearchTestImageRef & {
type: 'image';
})
| {
type: 'uploading';
key: string;
};
// Render persisted previews and upload placeholders in the same grid to avoid layout jumps.
const SearchTestImagePreviewList = ({
images,
uploadingCount,
onRemove
}: {
images: SearchTestImageRef[];
uploadingCount: number;
onRemove: (key: string) => void;
}) => {
if (images.length === 0 && uploadingCount === 0) return null;
return (
<Flex mb={3} gap={2} flexWrap={'wrap'}>
{images.map((image) => (
<SearchTestImagePreviewItem
key={image.key}
item={{ type: 'image', ...image }}
onRemove={onRemove}
/>
))}
{Array.from({ length: uploadingCount }).map((_, index) => (
<SearchTestImagePreviewItem key={index} item={{ type: 'uploading', key: `${index}` }} />
))}
</Flex>
);
};
export default React.memo(SearchTestImagePreviewList);
const SearchTestImagePreviewItem = React.memo(function SearchTestImagePreviewItem({
item,
onRemove
}: {
item: SearchTestImagePreviewListItem;
onRemove?: (key: string) => void;
}) {
return (
<Flex position={'relative'} overflow={'visible'} {...searchTestImageThumbProps}>
{item.type === 'image' ? (
<>
<Box
as={'img'}
src={item.previewUrl}
alt=""
w={'100%'}
h={'100%'}
objectFit={'cover'}
borderRadius={'6.66667px'}
/>
<Box
position={'absolute'}
right={'-7.5px'}
top={'-7.5px'}
w={'16.67px'}
h={'16.67px'}
display={'flex'}
alignItems={'center'}
justifyContent={'center'}
bg={'myGray.400'}
borderRadius={'50%'}
boxShadow={
'0px 6.66667px 6.66667px rgba(19, 51, 107, 0.1), 0px 0px 1.66667px rgba(19, 51, 107, 0.08)'
}
cursor={'pointer'}
onClick={() => onRemove?.(item.key)}
>
<MyIcon name={'common/closeLight'} w={'11.9px'} h={'11.9px'} color={'white'} />
</Box>
</>
) : (
<CircularProgress
value={28}
size={'46.67px'}
thickness={'8px'}
color={'primary.600'}
trackColor={'myGray.250'}
capIsRound
/>
)}
</Flex>
);
});
import React, { useMemo } from 'react';
import { Box, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { formatTimeToChatTime } from '@fastgpt/global/common/string/time';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import ImagePreviewToken from '@/components/core/dataset/ImagePreviewToken';
import {
useSearchTestStore,
type SearchTestStoreItemType
} from '@/web/core/dataset/store/searchTest';
const TestHistories = ({
datasetId,
datasetTestItem,
onSelect,
onClearSelect
}: {
datasetId: string;
datasetTestItem?: SearchTestStoreItemType;
onSelect: (item: SearchTestStoreItemType) => void;
onClearSelect: () => void;
}) => {
const { t } = useTranslation();
const { datasetTestList, delDatasetTestItemById } = useSearchTestStore();
// The store is shared across dataset pages; show only the current dataset's test records.
const testHistories = useMemo(
() => datasetTestList.filter((item) => item.datasetId === datasetId),
[datasetId, datasetTestList]
);
return (
<>
<Flex alignItems={'center'} color={'myGray.900'}>
<Box fontSize={'md'} fontWeight={500}>
{t('common:core.dataset.test.test history')}
</Box>
</Flex>
<Box mt={3} display={'flex'} flexDirection={'column'} gap={2}>
{testHistories.map((item) => (
<Flex
key={item.id}
position={'relative'}
py={2}
px={3}
alignItems={'center'}
borderColor={'borderColor.low'}
borderWidth={'1px'}
borderRadius={'md'}
_hover={{
borderColor: 'primary.300',
boxShadow: '1',
'& .delete': {
display: 'block'
},
'& .time': {
display: 'none'
}
}}
cursor={'pointer'}
fontSize={'sm'}
{...(item.id === datasetTestItem?.id && {
bg: 'primary.50'
})}
onClick={() => onSelect(item)}
>
<Box
flex={1}
mr={2}
wordBreak={'break-all'}
fontWeight={'400'}
display={'flex'}
alignItems={'center'}
flexWrap={'wrap'}
gap={1}
>
{!!item.text && <Box as={'span'}>{item.text}</Box>}
<ImagePreviewToken
images={item.queryImageRefs || []}
datasetId={datasetId}
containerProps={{
as: 'span',
display: 'inline-flex',
gap: 1
}}
tokenProps={{
px: 0,
py: 0,
border: 'none',
bg: 'transparent',
color: 'inherit'
}}
/>
</Box>
<Box className="time" flex={'0 0 auto'} fontSize={'xs'} color={'myGray.500'}>
{t(formatTimeToChatTime(item.time) as any).replace('#', ':')}
</Box>
<MyTooltip label={t('common:core.dataset.test.delete test history')}>
<Box className="delete" display={'none'} w={'0.8rem'} h={'0.8rem'} ml={1}>
<MyIcon
name={'delete'}
w={'0.8rem'}
_hover={{ color: 'red.600' }}
onClick={(e) => {
e.stopPropagation();
delDatasetTestItemById(item.id);
if (datasetTestItem?.id === item.id) {
onClearSelect();
}
}}
/>
</Box>
</MyTooltip>
</Flex>
))}
</Box>
</>
);
};
export default React.memo(TestHistories);
import React from 'react';
import { Box, Button, Flex, Textarea } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { SEARCH_TEST_IMAGE_UPLOAD_ENABLED } from '../constants';
import SearchTestImagePreviewList from './SearchTestImagePreviewList';
import type { SearchTestFormType, SearchTestImageRef } from '../type';
import type { UseFormRegister } from 'react-hook-form';
const TestInputPanel = ({
canSubmit,
canUseImageSearch,
datasetMaxToken,
isLoading,
onOpenImageSelector,
onOpenSelectMode,
onRemoveImage,
onSubmit,
queryImageRefs,
register,
showSearchTestImageEntry,
uploadingImageCount
}: {
canSubmit: boolean;
canUseImageSearch: boolean;
datasetMaxToken?: number;
isLoading: boolean;
onOpenImageSelector: () => void;
onOpenSelectMode: () => void;
onRemoveImage: (key: string) => void;
onSubmit: () => void;
queryImageRefs: SearchTestImageRef[];
register: UseFormRegister<SearchTestFormType>;
showSearchTestImageEntry: boolean;
uploadingImageCount: number;
}) => {
const { t } = useTranslation();
return (
<Box
display={'flex'}
flexDirection={'column'}
alignItems={'flex-start'}
gap={3}
alignSelf={'stretch'}
>
<Flex alignItems={'center'} alignSelf={'stretch'}>
<Box flex={1} fontWeight={500} color={'myGray.900'}>
{t('common:core.dataset.test.input_title')}
</Box>
<Button
variant={'whitePrimary'}
leftIcon={<MyIcon name={'common/settingLight'} w={'14px'} />}
size={'sm'}
fontWeight={500}
onClick={onOpenSelectMode}
>
{t('common:core.dataset.test.search_config')}
</Button>
</Flex>
<Box
border={'1px solid'}
borderColor={'borderColor.low'}
p={3}
borderRadius={'6px'}
minH={'220px'}
display={'flex'}
flexDirection={'column'}
bg={'white'}
alignSelf={'stretch'}
position={'relative'}
>
{showSearchTestImageEntry && (
<SearchTestImagePreviewList
images={queryImageRefs}
uploadingCount={uploadingImageCount}
onRemove={onRemoveImage}
/>
)}
<Textarea
flex={1}
minH={'140px'}
resize={'none'}
variant={'unstyled'}
py={0}
fontSize={'sm'}
lineHeight={'20px'}
color={'myGray.900'}
_placeholder={{
color: 'myGray.400'
}}
maxLength={datasetMaxToken}
placeholder={t('common:core.dataset.test.Test Text Placeholder')}
{...register('inputText')}
/>
{SEARCH_TEST_IMAGE_UPLOAD_ENABLED && (
<MyTooltip
label={canUseImageSearch ? '' : t('common:core.dataset.test.image_search_disabled_tip')}
>
<Box position={'absolute'} left={'12px'} bottom={'8px'}>
<Box
as={'button'}
w={'24px'}
h={'24px'}
p={0}
display={'flex'}
alignItems={'center'}
justifyContent={'center'}
bg={'transparent'}
border={'none'}
boxShadow={'none'}
_hover={{
bg: 'transparent'
}}
_active={{
bg: 'transparent'
}}
_disabled={{
bg: 'transparent',
opacity: 0.5,
cursor: 'not-allowed'
}}
disabled={!canUseImageSearch}
onClick={onOpenImageSelector}
aria-label={t('common:core.dataset.test.upload_image')}
>
<MyIcon name={'image'} w={'20px'} h={'20px'} color={'myGray.500'} flexShrink={0} />
</Box>
</Box>
</MyTooltip>
)}
</Box>
<Button
w={'100%'}
isLoading={isLoading}
isDisabled={!canSubmit || uploadingImageCount > 0}
onClick={onSubmit}
>
{t('common:core.dataset.test.Test')}
</Button>
</Box>
);
};
export default React.memo(TestInputPanel);
import React from 'react';
import { Box, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import QuoteItem from '@/components/core/dataset/QuoteItem';
import SearchParamsTip from '@/components/core/dataset/SearchParamsTip';
import type { SearchTestStoreItemType } from '@/web/core/dataset/store/searchTest';
const TestResults = ({ datasetTestItem }: { datasetTestItem?: SearchTestStoreItemType }) => {
const { t } = useTranslation();
if (!datasetTestItem?.results || datasetTestItem.results.length === 0) {
return (
<EmptyTip text={t('common:core.dataset.test.test result placeholder')} mt={[10, '20vh']} />
);
}
return (
<>
<Flex fontSize={'md'} color={'myGray.900'} alignItems={'center'} fontWeight={500}>
{t('common:core.dataset.test.Test params')}
</Flex>
<Box mt={3}>
<SearchParamsTip
searchMode={datasetTestItem.searchMode}
similarity={datasetTestItem.similarity}
limit={datasetTestItem.limit}
usingReRank={datasetTestItem.usingReRank}
usingExtensionQuery={!!datasetTestItem.queryExtensionModel}
queryExtensionModel={datasetTestItem.queryExtensionModel}
/>
</Box>
<Flex mt={5} mb={3} alignItems={'center'}>
<Flex fontSize={'md'} color={'myGray.900'} alignItems={'center'} fontWeight={500}>
{t('common:core.dataset.test.Test Result')}
</Flex>
<QuestionTip ml={1} label={t('common:core.dataset.test.test result tip')} />
<Box ml={2}>({datasetTestItem.duration})</Box>
</Flex>
<Box mt={1} gap={4}>
{datasetTestItem.results.map((item) => (
<Box key={item.id} p={3} borderRadius={'lg'} bg={'myGray.100'} _notLast={{ mb: 2 }}>
<QuoteItem quoteItem={item} canDownloadSource canEditData />
</Box>
))}
</Box>
</>
);
};
export default React.memo(TestResults);
import { imageFileType } from '@fastgpt/global/common/file/constants';
// Keep the disabled upload path in place so the search test flow can enable images by flag only.
export const SEARCH_TEST_IMAGE_UPLOAD_ENABLED = false;
export const MAX_SEARCH_TEST_IMAGE_COUNT = 10;
// Derive accepted extensions from the shared upload accept string to avoid frontend/backend drift.
export const IMAGE_EXTENSION_SET = new Set(
imageFileType
.split(',')
.map((item) => item.trim().replace('.', '').toLowerCase())
.filter(Boolean)
);
export const searchTestImageThumbProps = {
w: '80px',
h: '80px',
alignItems: 'center',
justifyContent: 'center',
bg: 'white',
border: '1.07143px solid',
borderColor: 'borderColor.low',
borderRadius: '8px',
boxShadow:
'0px 4.28571px 10.7143px rgba(19, 51, 107, 0.08), 0px 0px 1.07143px rgba(19, 51, 107, 0.08)'
} as const;
import { useState } from 'react';
import { useForm, useWatch } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import { postSearchText } from '@/web/core/dataset/api';
import {
useSearchTestStore,
type SearchTestStoreItemType
} from '@/web/core/dataset/store/searchTest';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import type { SearchDatasetTestResponse } from '@fastgpt/global/openapi/core/dataset/api';
import type { SearchTestFormType, SearchTestImageRef } from '../type';
export const useDatasetSearchTest = ({
datasetId,
queryImageRefs,
defaultModels
}: {
datasetId: string;
queryImageRefs: SearchTestImageRef[];
defaultModels: {
rerank?: {
model?: string;
};
llm?: {
model?: string;
};
};
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { pushDatasetTestItem } = useSearchTestStore();
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
const { control, getValues, setValue, register, handleSubmit } = useForm<SearchTestFormType>({
defaultValues: {
inputText: '',
searchParams: {
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false,
rerankModel: defaultModels?.rerank?.model,
rerankWeight: 0.5,
limit: 5000,
similarity: 0,
datasetSearchUsingExtensionQuery: false,
datasetSearchExtensionModel: defaultModels.llm?.model,
datasetSearchExtensionBg: ''
}
}
});
const inputText = useWatch({ control, name: 'inputText' });
const searchParams = getValues('searchParams');
const { runAsync: onTextTest, loading: textTestIsLoading } = useRequest(
({ inputText, searchParams }: SearchTestFormType) =>
postSearchText({
datasetId,
text: inputText.trim(),
// The backend resolves uploaded images by object key; previews stay client-side only.
queryImageUrls: queryImageRefs.map((item) => item.key),
...searchParams
}),
{
onSuccess(res: SearchDatasetTestResponse) {
if (!res || res.list.length === 0) {
return toast({
status: 'warning',
title: t('common:dataset.test.noResult')
});
}
const testItem: SearchTestStoreItemType = {
id: getNanoid(),
datasetId,
text: getValues('inputText').trim(),
time: new Date(),
results: res.list,
// Histories only need stable image keys; preview urls are regenerated by ImagePreviewToken.
queryImageRefs:
queryImageRefs.length > 0 ? queryImageRefs.map(({ key }) => ({ key })) : undefined,
duration: res.duration,
searchMode: res.searchMode,
usingReRank: res.usingReRank,
limit: res.limit,
similarity: res.similarity,
queryExtensionModel: res.queryExtensionModel
};
pushDatasetTestItem(testItem);
setDatasetTestItem(testItem);
}
}
);
return {
setDatasetTestItem,
currentDatasetTestItem: datasetTestItem?.datasetId === datasetId ? datasetTestItem : undefined,
inputText,
searchParams,
setValue,
register,
onSubmit: handleSubmit((data) => onTextTest(data)),
textTestIsLoading
};
};
import { useState } from 'react';
import { useTranslation } from 'next-i18next';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { getUploadSearchTestImagePresignedUrl } from '@/web/core/dataset/api/file';
import { useUserStore } from '@/web/support/user/useUserStore';
import { imageFileType } from '@fastgpt/global/common/file/constants';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { putFileToS3 } from '@fastgpt/web/common/file/utils';
import { useToast } from '@fastgpt/web/hooks/useToast';
import {
IMAGE_EXTENSION_SET,
MAX_SEARCH_TEST_IMAGE_COUNT,
SEARCH_TEST_IMAGE_UPLOAD_ENABLED
} from '../constants';
import type { SearchTestImageRef } from '../type';
export const useSearchTestImages = ({
datasetId,
canUseImageSearch,
uploadFileMaxSize
}: {
datasetId: string;
canUseImageSearch: boolean;
uploadFileMaxSize?: number;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { teamPlanStatus, initTeamPlanStatus } = useUserStore();
const { File: ImageFileSelector, onOpen: onOpenImageSelector } = useSelectFile({
fileType: imageFileType,
multiple: true,
maxCount: MAX_SEARCH_TEST_IMAGE_COUNT
});
const [queryImageRefs, setQueryImageRefs] = useState<SearchTestImageRef[]>([]);
const [uploadingImageCount, setUploadingImageCount] = useState(0);
// Feature flag lets the UI stay wired while image search upload is being rolled out.
const showSearchTestImageEntry = SEARCH_TEST_IMAGE_UPLOAD_ENABLED && canUseImageSearch;
const onSelectFile = async (files: File[]) => {
const imageFiles = files.filter((file) => {
const extension = file.name.split('.').pop()?.toLowerCase();
return !!extension && IMAGE_EXTENSION_SET.has(extension);
});
if (imageFiles.length < files.length) {
toast({
status: 'warning',
title: t('chat:unsupported_file_type')
});
}
// Prefer live team limits; fall back to frontend config so validation still works offline.
const planStatus =
teamPlanStatus ||
(await initTeamPlanStatus()
.then(() => useUserStore.getState().teamPlanStatus)
.catch(() => undefined));
const maxImageSize =
(planStatus?.standard?.maxUploadFileSize ?? uploadFileMaxSize ?? 500) * 1024 * 1024;
const validImageFiles = imageFiles.filter((file) => file.size <= maxImageSize);
if (validImageFiles.length < imageFiles.length) {
toast({
status: 'warning',
title: t('file:some_file_size_exceeds_limit', {
maxSize: formatFileSize(maxImageSize)
})
});
}
if (queryImageRefs.length + validImageFiles.length > MAX_SEARCH_TEST_IMAGE_COUNT) {
toast({
status: 'warning',
title: t('common:core.dataset.test.max_images_tip')
});
}
// Trim the selected batch instead of rejecting all files when the max count is exceeded.
const uploadFiles = validImageFiles.slice(
0,
Math.max(MAX_SEARCH_TEST_IMAGE_COUNT - queryImageRefs.length, 0)
);
if (uploadFiles.length === 0) return;
setUploadingImageCount(uploadFiles.length);
try {
const uploadedImages = await Promise.all(
uploadFiles.map(async (file) => {
const { url, key, headers, maxSize, previewUrl } =
await getUploadSearchTestImagePresignedUrl({
datasetId,
filename: file.name
});
await putFileToS3({
url,
headers,
file,
maxSize,
t
});
return { key, previewUrl };
})
);
setQueryImageRefs((state) => [...state, ...uploadedImages]);
} catch {
toast({
status: 'warning',
title: t('common:upload_file_error')
});
} finally {
setUploadingImageCount(0);
}
};
return {
ImageFileSelector,
queryImageRefs,
uploadingImageCount,
showSearchTestImageEntry,
onOpenImageSelector,
onSelectFile,
removeImage: (key: string) =>
setQueryImageRefs((state) => state.filter((item) => item.key !== key))
};
};
import React from 'react';
import dynamic from 'next/dynamic';
import { Box, useDisclosure } from '@chakra-ui/react';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { SEARCH_TEST_IMAGE_UPLOAD_ENABLED } from './constants';
import TestInputPanel from './components/TestInputPanel';
import TestHistories from './components/TestHistories';
import TestResults from './components/TestResults';
import { useDatasetSearchTest } from './hooks/useDatasetSearchTest';
import { useSearchTestImages } from './hooks/useSearchTestImages';
const DatasetParamsModal = dynamic(() => import('@/components/core/app/DatasetParamsModal'));
const Test = ({ datasetId }: { datasetId: string }) => {
const { defaultModels, feConfigs } = useSystemStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
// Image search is only meaningful when the dataset has a vision vector model or VLM configured.
const canUseImageSearch = !!datasetDetail.vectorModel?.vision || !!datasetDetail.vlmModel;
const {
ImageFileSelector,
queryImageRefs,
uploadingImageCount,
showSearchTestImageEntry,
onOpenImageSelector,
onSelectFile,
removeImage
} = useSearchTestImages({
datasetId,
canUseImageSearch,
uploadFileMaxSize: feConfigs?.uploadFileMaxSize
});
const {
currentDatasetTestItem,
inputText,
searchParams,
setValue,
register,
onSubmit,
textTestIsLoading,
setDatasetTestItem
} = useDatasetSearchTest({
datasetId,
queryImageRefs: showSearchTestImageEntry ? queryImageRefs : [],
defaultModels
});
const {
isOpen: isOpenSelectMode,
onOpen: onOpenSelectMode,
onClose: onCloseSelectMode
} = useDisclosure();
// Text and image can both be search inputs; keep image-only test available for multimodal datasets.
const canSubmit = !!inputText?.trim() || (showSearchTestImageEntry && queryImageRefs.length > 0);
return (
<Box h={'100%'} display={['block', 'flex']}>
<Box
h={['auto', '100%']}
display={['block', 'flex']}
flexDirection={'column'}
flex={['unset', '0 0 468px']}
w={['100%', '468px']}
p={4}
gap={6}
borderRightWidth={['0', '1px']}
borderRightStyle={'solid'}
borderRightColor={'borderColor.low'}
>
<TestInputPanel
canSubmit={canSubmit}
canUseImageSearch={canUseImageSearch}
datasetMaxToken={datasetDetail.vectorModel?.maxToken}
isLoading={textTestIsLoading}
onOpenImageSelector={onOpenImageSelector}
onOpenSelectMode={onOpenSelectMode}
onRemoveImage={removeImage}
onSubmit={onSubmit}
queryImageRefs={queryImageRefs}
register={register}
showSearchTestImageEntry={showSearchTestImageEntry}
uploadingImageCount={uploadingImageCount}
/>
<Box overflow={'overlay'} display={['none', 'block']}>
<TestHistories
datasetId={datasetId}
datasetTestItem={currentDatasetTestItem}
onSelect={setDatasetTestItem}
onClearSelect={() => setDatasetTestItem(undefined)}
/>
</Box>
</Box>
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={'1 0 0'} bg={'white'}>
<TestResults datasetTestItem={currentDatasetTestItem} />
</Box>
{isOpenSelectMode && (
<DatasetParamsModal
{...searchParams}
maxTokens={20000}
onClose={onCloseSelectMode}
onSuccess={(e) => {
setValue('searchParams', {
...searchParams,
...e
});
}}
/>
)}
{SEARCH_TEST_IMAGE_UPLOAD_ENABLED && <ImageFileSelector onSelect={onSelectFile} />}
</Box>
);
};
export default React.memo(Test);
import type { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
export type SearchTestFormType = {
inputText: string;
searchParams: {
searchMode: DatasetSearchModeEnum;
embeddingWeight?: number;
usingReRank?: boolean;
rerankModel?: string;
rerankWeight?: number;
similarity?: number;
limit?: number;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
};
};
export type SearchTestImageRef = {
key: string;
previewUrl: string;
};
import React from 'react';
import { Box, Button, Flex, Spinner, Textarea, type TextareaProps } from '@chakra-ui/react';
import type { UseFormRegister } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyImage from '@/components/MyImage/index';
import { TabEnum, type InputDataType } from './useInputDataModal';
const dataTextareaProps: TextareaProps = {
resize: 'both',
flex: '1 0 0',
bg: 'white',
borderRadius: '6px',
border: '1px solid',
borderColor: 'borderColor.low',
p: '8px 12px',
color: 'myGray.900',
_focus: {
borderColor: 'primary.500',
boxShadow: 'focus',
bg: 'white'
},
sx: {
'&::-webkit-scrollbar-thumb': {
background: 'var(--chakra-colors-myGray-150) !important',
transition: 'background 1s',
marginLeft: '5px'
},
'&::-webkit-scrollbar-thumb:hover': {
background: 'var(--chakra-colors-myGray-250) !important'
}
}
};
const getDataTextareaProps = (canWrite: boolean): TextareaProps => ({
...dataTextareaProps,
isReadOnly: !canWrite,
cursor: canWrite ? 'text' : 'default',
_readOnly: {
bg: 'myGray.25',
color: 'myGray.600'
}
});
const DataInputPanel = ({
canWrite,
currentTab,
imagePreivewUrl,
isImporting,
isUpdating,
isIndexEditing,
register,
submitData
}: {
canWrite: boolean;
currentTab?: TabEnum;
imagePreivewUrl?: string;
isImporting: boolean;
isUpdating: boolean;
isIndexEditing: boolean;
register: UseFormRegister<InputDataType>;
submitData: () => void;
}) => {
const { t } = useTranslation();
const textareaProps = getDataTextareaProps(canWrite);
const isSubmitting = isImporting || isUpdating;
const isSubmitDisabled = !canWrite || isIndexEditing;
const iconColor = isSubmitDisabled ? 'myGray.400' : 'primary.600';
return (
<Flex
flexDir={'column'}
gap={'8px'}
flex={'1 0 0'}
w={['100%', 0]}
overflow={['unset', 'auto']}
>
<Flex
flexDir={'column'}
flex={currentTab === TabEnum.image ? '0 0 201px' : '1 0 0'}
h={currentTab === TabEnum.image ? '201px' : 0}
minH={0}
gap={'8px'}
>
{currentTab === TabEnum.image && (
<>
<FormLabel required h={'32px'} py={'6px'}>
{t('file:image')}
</FormLabel>
<Box flex={'1 0 0'} h={0} w="100%">
<Box
height="100%"
position="relative"
border={'1px solid'}
borderColor={'borderColor.low'}
borderRadius={'6px'}
bg={'myGray.25'}
p={'8px'}
>
<MyImage
src={imagePreivewUrl}
h="100%"
w="100%"
objectFit="cover"
borderRadius={'2px'}
alt={t('file:Image_Preview')}
/>
</Box>
</Box>
</>
)}
{(currentTab === TabEnum.chunk || currentTab === TabEnum.qa) && (
<>
<FormLabel required h={'20px'}>
{currentTab === TabEnum.chunk
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
<Textarea
{...textareaProps}
{...register(`q`, {
required: true
})}
/>
</>
)}
</Flex>
{currentTab === TabEnum.qa && (
<Flex flexDir={'column'} flex={'1 0 0'} h={0} minH={0} gap={'8px'}>
<FormLabel required h={'20px'}>
{t('common:dataset_data_input_a')}
</FormLabel>
<Textarea {...textareaProps} {...register('a', { required: true })} />
</Flex>
)}
{currentTab === TabEnum.image && (
<Flex flexDir={'column'} flex={'1 0 0'} h={0} minH={0} gap={'8px'}>
<FormLabel required h={'32px'} py={'6px'}>
{t('file:image_description')}
</FormLabel>
<Textarea
{...textareaProps}
placeholder={t('file:image_description_tip')}
{...register('q', {
required: true
})}
/>
</Flex>
)}
<Button
h={'32px'}
minH={'32px'}
w={'100%'}
bg={'myGray.150'}
color={'primary.700'}
borderRadius={'6px'}
fontSize={'12px'}
lineHeight={'16px'}
fontWeight={'500'}
letterSpacing={'0.5px'}
_hover={{ bg: 'myGray.200' }}
_disabled={{
bg: 'myGray.100',
color: 'myGray.400',
cursor: 'not-allowed',
opacity: 1,
_hover: {
bg: 'myGray.100'
}
}}
rightIcon={
isSubmitting ? (
<Spinner w={'16px'} h={'16px'} color={iconColor} />
) : (
<MyIcon name={'common/rightArrowLight'} w={'16px'} color={iconColor} />
)
}
isDisabled={isSubmitDisabled}
aria-busy={isSubmitting}
onClick={() => {
if (isSubmitting || isSubmitDisabled) return;
submitData();
}}
>
{t('dataset:generate_index')}
</Button>
</Flex>
);
};
export default React.memo(DataInputPanel);
import React from 'react';
import { Box, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import MyModal from '@fastgpt/web/components/v2/common/MyModal';
import MyBox from '@fastgpt/web/components/common/MyBox';
import DataInputPanel from './DataInputPanel';
import IndexInputPanel from './IndexInputPanel';
import { TabEnum, useInputDataModal, type InputDataType } from './useInputDataModal';
const InputDataModal = ({
collectionId,
dataId,
defaultValue,
onClose,
onSuccess
}: {
collectionId: string;
dataId?: string;
defaultValue?: { q?: string; a?: string; imagePreivewUrl?: string };
onClose: () => void;
onSuccess: (data: InputDataType & { dataId: string }) => void;
}) => {
const { t } = useTranslation();
const {
collection,
currentTab,
deletingIndexClientId,
editingIndexClientId,
focusIndexClientId,
imagePreivewUrl,
indexes,
initLoading,
isDeletingIndex,
isImporting,
isUpdating,
maxToken,
register,
showTabs,
submitData,
watchedIndexes,
onDeleteIndex,
onSaveIndex,
prependCustomIndex,
setCurrentTab,
clearFocusIndexClientId,
markEditingIndex,
clearEditingIndex,
markDeletingIndex,
updateIndexFold
} = useInputDataModal({
collectionId,
dataId,
defaultValue,
onSuccess
});
return (
<MyModal
isOpen={true}
isCentered
w={['calc(100vw - 32px)', '800px']}
onClose={onClose}
closeOnOverlayClick={false}
maxW={['calc(100vw - 32px)', '800px']}
h={['auto', currentTab === TabEnum.image ? '584px' : '620px']}
maxH={['90vh', 'calc(100vh - 48px)']}
borderRadius={'10px'}
bg={'white'}
boxShadow={'3.5'}
contentPx={[5, '32px']}
contentPy={[5, '32px']}
headerPx={0}
title={
<Box
className={'textEllipsis'}
wordBreak={'break-all'}
fontSize={['xl', '20px']}
lineHeight={['28px', '26px']}
maxW={['calc(100vw - 96px)', '680px']}
fontWeight={'500'}
letterSpacing={'0.15px'}
color={'black'}
whiteSpace={'nowrap'}
overflow={'hidden'}
textOverflow={'ellipsis'}
>
{collection.sourceName || t('common:unknow_source')}
</Box>
}
>
<MyBox
display={'flex'}
flexDir={'column'}
isLoading={initLoading}
h={'100%'}
minH={['300px', showTabs ? '506px' : '450px']}
overflow={'hidden'}
>
{!initLoading && (
<Flex
flexDir={'column'}
gap={'24px'}
w={'100%'}
h={['auto', showTabs ? '506px' : '450px']}
minH={0}
>
{showTabs && (
<Flex
h={'32px'}
gap={'16px'}
borderBottom={'1px solid'}
borderColor={'borderColor.low'}
flexShrink={0}
>
{[
{ label: t('common:dataset_data_input_chunk'), value: TabEnum.chunk },
{ label: t('common:dataset_data_input_qa'), value: TabEnum.qa }
].map((item) => {
const isActive = currentTab === item.value;
return (
<Flex
key={item.value}
alignItems={'center'}
justifyContent={'center'}
h={'32px'}
px={'4px'}
borderBottom={'1.5px solid'}
borderColor={isActive ? 'primary.600' : 'transparent'}
color={isActive ? 'primary.700' : 'myGray.500'}
fontSize={'16px'}
lineHeight={'24px'}
fontWeight={'500'}
letterSpacing={'0.15px'}
cursor={'pointer'}
onClick={() => setCurrentTab(item.value)}
>
{item.label}
</Flex>
);
})}
</Flex>
)}
<Flex
flex={'1 0 0'}
h={['auto', '450px']}
gap={'32px'}
flexDir={['column', 'row']}
minH={0}
>
<DataInputPanel
canWrite={collection.permission.hasWritePer}
currentTab={currentTab}
imagePreivewUrl={imagePreivewUrl}
isImporting={isImporting}
isUpdating={isUpdating}
isIndexEditing={!!editingIndexClientId}
register={register}
submitData={submitData}
/>
<IndexInputPanel
canWrite={collection.permission.hasWritePer}
deletingIndexClientId={deletingIndexClientId}
focusIndexClientId={focusIndexClientId}
indexes={indexes}
isDeletingIndex={isDeletingIndex}
maxToken={maxToken}
register={register}
watchedIndexes={watchedIndexes}
onAddIndex={prependCustomIndex}
onDeleteIndex={onDeleteIndex}
onSaveIndex={onSaveIndex}
onAutoFocusIndex={clearFocusIndexClientId}
onIndexFocus={markEditingIndex}
onIndexBlur={clearEditingIndex}
onDeleteIntent={markDeletingIndex}
updateIndexFold={updateIndexFold}
/>
</Flex>
</Flex>
)}
</MyBox>
</MyModal>
);
};
export default React.memo(InputDataModal);
import React, { useMemo } from 'react'; import React, { useMemo } from 'react';
import { Box, Flex, Button, ModalFooter, ModalBody, Input, HStack } from '@chakra-ui/react'; import { Box, Flex, Button, Input, HStack } from '@chakra-ui/react';
import { useForm } from 'react-hook-form'; import { useForm } from 'react-hook-form';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { useSystemStore } from '@/web/common/system/useSystemStore'; import { useSystemStore } from '@/web/common/system/useSystemStore';
import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { useRequest } from '@fastgpt/web/hooks/useRequest';
import Avatar from '@fastgpt/web/components/common/Avatar'; import Avatar from '@fastgpt/web/components/common/Avatar';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import MyModal from '@fastgpt/web/components/common/MyModal'; import MyModal from '@fastgpt/web/components/v2/common/MyModal';
import { postCreateDataset } from '@/web/core/dataset/api'; import { postCreateDataset } from '@/web/core/dataset/api';
import type { CreateDatasetBody } from '@fastgpt/global/openapi/core/dataset/api'; import type { CreateDatasetBody } from '@fastgpt/global/openapi/core/dataset/api';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import { DatasetTypeEnum, DatasetTypeMap } from '@fastgpt/global/core/dataset/constants'; import { DatasetTypeEnum, DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
import AIModelSelector from '@/components/Select/AIModelSelector'; import AIModelSelector from '@/components/Select/AIModelSelector';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import ComplianceTip from '@/components/common/ComplianceTip/index'; import ComplianceTip from '@/components/common/ComplianceTip/index';
import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIcon from '@fastgpt/web/components/common/Icon';
...@@ -42,7 +41,6 @@ const CreateModal = ({ ...@@ -42,7 +41,6 @@ const CreateModal = ({
const { t } = useTranslation(); const { t } = useTranslation();
const router = useRouter(); const router = useRouter();
const { defaultModels, embeddingModelList, llmModelList, getVlmModelList } = useSystemStore(); const { defaultModels, embeddingModelList, llmModelList, getVlmModelList } = useSystemStore();
const { isPc } = useSystem();
const filterNotHiddenVectorModelList = embeddingModelList.filter((item) => !item.hidden); const filterNotHiddenVectorModelList = embeddingModelList.filter((item) => !item.hidden);
...@@ -66,6 +64,11 @@ const CreateModal = ({ ...@@ -66,6 +64,11 @@ const CreateModal = ({
const vectorModel = watch('vectorModel'); const vectorModel = watch('vectorModel');
const agentModel = watch('agentModel'); const agentModel = watch('agentModel');
const vlmModel = watch('vlmModel'); const vlmModel = watch('vlmModel');
const showApiDatasetForm =
type === DatasetTypeEnum.apiDataset ||
type === DatasetTypeEnum.feishu ||
type === DatasetTypeEnum.yuque ||
type === DatasetTypeEnum.dingtalk;
const { Component: AvatarUploader, handleFileSelectorOpen: handleAvatarSelectorOpen } = const { Component: AvatarUploader, handleFileSelectorOpen: handleAvatarSelectorOpen } =
useUploadAvatar(getUploadAvatarPresignedUrl, { useUploadAvatar(getUploadAvatarPresignedUrl, {
...@@ -88,28 +91,25 @@ const CreateModal = ({ ...@@ -88,28 +91,25 @@ const CreateModal = ({
return ( return (
<MyModal <MyModal
title={
<Flex alignItems={'center'} ml={-3}>
<Avatar
w={'20px'}
h={'20px'}
borderRadius={'xs'}
src={DatasetTypeMap[type].avatar}
pr={'10px'}
/>
{t('common:core.dataset.Create dataset', { name: t(DatasetTypeMap[type].label) })}
</Flex>
}
isOpen isOpen
onClose={onClose} onClose={onClose}
isCentered={!isPc} size={'md'}
w={'490px'} iconSrc={DatasetTypeMap[type].avatar}
title={t('dataset:create_dataset_title', { name: t(DatasetTypeMap[type].label) })}
contentPx={'32px'}
contentPy={'32px'}
borderRadius={'10px'}
> >
<ModalBody py={6} px={9}> <Flex
<Box> flexDirection={'column'}
alignItems={'flex-start'}
minH={showApiDatasetForm ? undefined : '338px'}
>
<Flex w={'100%'} flexDirection={'column'} gap={4}>
<Box w={'100%'}>
<Flex justify={'space-between'}> <Flex justify={'space-between'}>
<Box color={'myGray.900'} fontWeight={500} fontSize={'sm'}> <Box color={'myGray.900'} fontWeight={500} fontSize={'sm'}>
{t('common:input_name')} {t('common:Name')}
</Box> </Box>
{DatasetTypeMap[type]?.courseUrl && ( {DatasetTypeMap[type]?.courseUrl && (
<Flex <Flex
...@@ -138,11 +138,12 @@ const CreateModal = ({ ...@@ -138,11 +138,12 @@ const CreateModal = ({
/> />
</MyTooltip> </MyTooltip>
<Input <Input
ml={3} ml={4}
flex={1} flex={1}
autoFocus autoFocus
bg={'myWhite.600'} bg={'myWhite.600'}
placeholder={t('common:Name')} fontSize={'14px'}
placeholder={t('dataset:dataset_name_placeholder')}
maxLength={30} maxLength={30}
{...register('name', { {...register('name', {
required: true required: true
...@@ -152,7 +153,7 @@ const CreateModal = ({ ...@@ -152,7 +153,7 @@ const CreateModal = ({
</Box> </Box>
<Flex <Flex
mt={6} w={'100%'}
alignItems={['flex-start', 'center']} alignItems={['flex-start', 'center']}
justify={'space-between'} justify={'space-between'}
flexDir={['column', 'row']} flexDir={['column', 'row']}
...@@ -185,7 +186,7 @@ const CreateModal = ({ ...@@ -185,7 +186,7 @@ const CreateModal = ({
</Flex> </Flex>
<Flex <Flex
mt={6} w={'100%'}
alignItems={['flex-start', 'center']} alignItems={['flex-start', 'center']}
justify={'space-between'} justify={'space-between'}
flexDir={['column', 'row']} flexDir={['column', 'row']}
...@@ -217,13 +218,14 @@ const CreateModal = ({ ...@@ -217,13 +218,14 @@ const CreateModal = ({
</Flex> </Flex>
<Flex <Flex
mt={6} w={'100%'}
alignItems={['flex-start', 'center']} alignItems={['flex-start', 'center']}
justify={'space-between'} justify={'space-between'}
flexDir={['column', 'row']} flexDir={['column', 'row']}
> >
<HStack <HStack
spacing={1} spacing={1}
alignItems={'center'}
flex={['', '0 0 110px']} flex={['', '0 0 110px']}
fontSize={'sm'} fontSize={'sm'}
color={'myGray.900'} color={'myGray.900'}
...@@ -231,6 +233,7 @@ const CreateModal = ({ ...@@ -231,6 +233,7 @@ const CreateModal = ({
pb={['12px', '0']} pb={['12px', '0']}
> >
<Box>{t('dataset:vllm_model')}</Box> <Box>{t('dataset:vllm_model')}</Box>
<QuestionTip label={t('dataset:vllm_model_tip')} />
</HStack> </HStack>
<Box w={['100%', '300px']}> <Box w={['100%', '300px']}>
<AIModelSelector <AIModelSelector
...@@ -246,23 +249,39 @@ const CreateModal = ({ ...@@ -246,23 +249,39 @@ const CreateModal = ({
/> />
</Box> </Box>
</Flex> </Flex>
</Flex>
{/* @ts-ignore */} {showApiDatasetForm && (
<ApiDatasetForm type={type} form={form} /> <Box
</ModalBody> mt={4}
w={'100%'}
sx={{
'& > *:first-of-type': {
mt: '0 !important'
}
}}
>
<ApiDatasetForm type={type} form={form} controlWidth={['100%', '300px']} />
</Box>
)}
<ModalFooter px={9}> <Flex mt={6} w={'100%'} justifyContent={'flex-end'} gap={3}>
<Button variant={'whiteBase'} mr={3} onClick={onClose}> <Button variant={'whiteBase'} fontSize={'12px'} onClick={onClose}>
{t('common:Close')} {t('common:Close')}
</Button> </Button>
<Button isLoading={creating} onClick={handleSubmit((data) => onclickCreate(data))}> <Button
{t('common:comfirn_create')} fontSize={'12px'}
isLoading={creating}
onClick={handleSubmit((data) => onclickCreate(data))}
>
{t('common:Create')}
</Button> </Button>
</ModalFooter> </Flex>
<ComplianceTip pb={6} pt={0} px={9} type={'dataset'} /> <ComplianceTip pb={0} pt={0} px={0} type={'dataset'} />
<AvatarUploader /> <AvatarUploader />
</Flex>
</MyModal> </MyModal>
); );
}; };
......
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth'; import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
import { deleteDatasetData } from '@/service/core/dataset/data/controller'; import { deleteDatasetData } from '@/service/core/dataset/data/data';
import { NextAPI } from '@/service/middleware/entry'; import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { addAuditLog } from '@fastgpt/service/support/user/audit/util';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { DeleteDatasetDataQuerySchema } from '@fastgpt/global/openapi/core/dataset/data/api'; import {
DeleteDatasetDataQuerySchema,
DeleteDatasetDataResponseSchema,
type DeleteDatasetDataResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
async function handler(req: ApiRequestProps) { async function handler(req: ApiRequestProps): Promise<DeleteDatasetDataResponse> {
const { id: dataId } = DeleteDatasetDataQuerySchema.parse(req.query); const { id: dataId } = DeleteDatasetDataQuerySchema.parse(req.query);
// 凭证校验 // 凭证校验
...@@ -34,7 +38,7 @@ async function handler(req: ApiRequestProps) { ...@@ -34,7 +38,7 @@ async function handler(req: ApiRequestProps) {
} }
}); });
})(); })();
return 'success'; return DeleteDatasetDataResponseSchema.parse('success');
} }
export default NextAPI(handler); export default NextAPI(handler);
import { createDatasetDataIndex } from '@/service/core/dataset/data/dataIndex';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { NextAPI } from '@/service/middleware/entry';
import {
CreateDatasetDataIndexBodySchema,
DatasetDataIndexResponseSchema,
type DatasetDataIndexResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { addAuditLog, getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
async function handler(req: ApiRequestProps): Promise<DatasetDataIndexResponse> {
const { dataId, type, text } = CreateDatasetDataIndexBodySchema.parse(req.body);
const { datasetData, tmbId, teamId, collection } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: WritePermissionVal
});
const { index, tokens } = await createDatasetDataIndex({
data: datasetData,
type,
text,
model: collection.dataset.vectorModel
});
if (tokens > 0) {
pushGenerateVectorUsage({
teamId,
tmbId,
inputTokens: tokens,
model: collection.dataset.vectorModel
});
}
addAuditLog({
tmbId,
teamId,
event: AuditEventEnum.UPDATE_DATA,
params: {
collectionName: collection.name,
datasetName: collection.dataset?.name || '',
datasetType: getI18nDatasetType(collection.dataset?.type || '')
}
});
return DatasetDataIndexResponseSchema.parse({
index
});
}
export default NextAPI(handler);
import { deleteDatasetDataIndex } from '@/service/core/dataset/data/dataIndex';
import { NextAPI } from '@/service/middleware/entry';
import {
DeleteDatasetDataIndexBodySchema,
DeleteDatasetDataIndexResponseSchema,
type DeleteDatasetDataIndexResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { addAuditLog, getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
async function handler(req: ApiRequestProps): Promise<DeleteDatasetDataIndexResponse> {
const { dataId, indexDataId } = DeleteDatasetDataIndexBodySchema.parse(req.body);
const { datasetData, tmbId, teamId, collection } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: WritePermissionVal
});
await deleteDatasetDataIndex({
data: datasetData,
indexDataId
});
addAuditLog({
tmbId,
teamId,
event: AuditEventEnum.UPDATE_DATA,
params: {
collectionName: collection.name,
datasetName: collection.dataset?.name || '',
datasetType: getI18nDatasetType(collection.dataset?.type || '')
}
});
return DeleteDatasetDataIndexResponseSchema.parse({});
}
export default NextAPI(handler);
import { updateDatasetDataIndex } from '@/service/core/dataset/data/dataIndex';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { NextAPI } from '@/service/middleware/entry';
import {
UpdateDatasetDataIndexBodySchema,
DatasetDataIndexResponseSchema,
type DatasetDataIndexResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { addAuditLog, getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
async function handler(req: ApiRequestProps): Promise<DatasetDataIndexResponse> {
const { dataId, indexDataId, type, text } = UpdateDatasetDataIndexBodySchema.parse(req.body);
const { datasetData, tmbId, teamId, collection } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: WritePermissionVal
});
const { index, tokens } = await updateDatasetDataIndex({
data: datasetData,
indexDataId,
type,
text,
model: collection.dataset.vectorModel
});
if (tokens > 0) {
pushGenerateVectorUsage({
teamId,
tmbId,
inputTokens: tokens,
model: collection.dataset.vectorModel
});
}
addAuditLog({
tmbId,
teamId,
event: AuditEventEnum.UPDATE_DATA,
params: {
collectionName: collection.name,
datasetName: collection.dataset?.name || '',
datasetType: getI18nDatasetType(collection.dataset?.type || '')
}
});
return DatasetDataIndexResponseSchema.parse({
index
});
}
export default NextAPI(handler);
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
*/ */
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model'; import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
import { hasSameValue } from '@/service/core/dataset/data/utils'; import { hasSameValue } from '@/service/core/dataset/data/utils';
import { insertData2Dataset } from '@/service/core/dataset/data/controller'; import { createDatasetData } from '@/service/core/dataset/data/data';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller'; import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push'; import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
...@@ -41,7 +41,7 @@ async function handler(req: ApiRequestProps): Promise<InsertDataResponse> { ...@@ -41,7 +41,7 @@ async function handler(req: ApiRequestProps): Promise<InsertDataResponse> {
const [ const [
{ {
dataset: { _id: datasetId, vectorModel, agentModel }, dataset: { _id: datasetId, vectorModel },
indexPrefixTitle, indexPrefixTitle,
name name
} }
...@@ -64,7 +64,7 @@ async function handler(req: ApiRequestProps): Promise<InsertDataResponse> { ...@@ -64,7 +64,7 @@ async function handler(req: ApiRequestProps): Promise<InsertDataResponse> {
a: formatA a: formatA
}); });
const { insertId, tokens } = await insertData2Dataset({ const { insertId, tokens } = await createDatasetData({
teamId, teamId,
tmbId, tmbId,
datasetId, datasetId,
......
import { updateData2Dataset } from '@/service/core/dataset/data/controller'; import {
updateDatasetDataByIndexes,
updateDatasetDataDefaultIndexes
} from '@/service/core/dataset/data/data';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push'; import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { NextAPI } from '@/service/middleware/entry'; import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
...@@ -7,10 +10,15 @@ import { type ApiRequestProps } from '@fastgpt/service/type/next'; ...@@ -7,10 +10,15 @@ import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { addAuditLog } from '@fastgpt/service/support/user/audit/util'; import { addAuditLog } from '@fastgpt/service/support/user/audit/util';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants'; import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util'; import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { UpdateDatasetDataBodySchema } from '@fastgpt/global/openapi/core/dataset/data/api'; import {
UpdateDatasetDataBodySchema,
UpdateDatasetDataResponseSchema,
type UpdateDatasetDataResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
async function handler(req: ApiRequestProps) { async function handler(req: ApiRequestProps): Promise<UpdateDatasetDataResponse> {
const { dataId, q, a, indexes = [] } = UpdateDatasetDataBodySchema.parse(req.body); const { dataId, q, a, indexes } = UpdateDatasetDataBodySchema.parse(req.body);
const hasIndexes = Object.hasOwn(req.body, 'indexes');
// auth data permission // auth data permission
const { const {
...@@ -21,7 +29,8 @@ async function handler(req: ApiRequestProps) { ...@@ -21,7 +29,8 @@ async function handler(req: ApiRequestProps) {
}, },
teamId, teamId,
tmbId, tmbId,
collection collection,
datasetData
} = await authDatasetData({ } = await authDatasetData({
req, req,
authToken: true, authToken: true,
...@@ -30,22 +39,46 @@ async function handler(req: ApiRequestProps) { ...@@ -30,22 +39,46 @@ async function handler(req: ApiRequestProps) {
per: WritePermissionVal per: WritePermissionVal
}); });
if (q || a || indexes.length > 0) { if (hasIndexes) {
const { tokens } = await updateData2Dataset({ const { tokens } = await updateDatasetDataByIndexes({
dataId, dataId,
q, q,
a, a,
indexes, indexes: indexes ?? [],
model: vectorModel, model: vectorModel,
indexPrefix: indexPrefixTitle ? `# ${name}` : undefined indexPrefix: indexPrefixTitle ? `# ${name}` : undefined
}); });
if (tokens > 0) {
pushGenerateVectorUsage({ pushGenerateVectorUsage({
teamId, teamId,
tmbId, tmbId,
inputTokens: tokens, inputTokens: tokens,
model: vectorModel model: vectorModel
}); });
}
} else {
const nextQ = q || datasetData.q || '';
const nextA = a ?? datasetData.a ?? '';
const { tokens } = await updateDatasetDataDefaultIndexes({
dataId,
q: nextQ,
a: nextA,
model: vectorModel,
indexSize: collection.indexSize,
indexPrefix: indexPrefixTitle ? `# ${name}` : undefined
});
if (tokens > 0) {
pushGenerateVectorUsage({
teamId,
tmbId,
inputTokens: tokens,
model: vectorModel
});
}
}
(() => { (() => {
addAuditLog({ addAuditLog({
...@@ -59,7 +92,8 @@ async function handler(req: ApiRequestProps) { ...@@ -59,7 +92,8 @@ async function handler(req: ApiRequestProps) {
} }
}); });
})(); })();
}
return UpdateDatasetDataResponseSchema.parse({});
} }
export default NextAPI(handler); export default NextAPI(handler);
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { addHours } from 'date-fns';
import { S3Buckets } from '@fastgpt/service/common/s3/config/constants';
import { isS3ObjectKey, jwtSignS3DownloadToken } from '@fastgpt/service/common/s3/utils';
import {
GetSearchTestImagePreviewUrlsBodySchema,
GetSearchTestImagePreviewUrlsResponseSchema,
type GetSearchTestImagePreviewUrlsBody,
type GetSearchTestImagePreviewUrlsResponse
} from '@fastgpt/global/openapi/core/dataset/file/api';
async function handler(
req: ApiRequestProps<GetSearchTestImagePreviewUrlsBody>
): Promise<GetSearchTestImagePreviewUrlsResponse> {
const { datasetId, keys } = GetSearchTestImagePreviewUrlsBodySchema.parse(req.body);
const { teamId } = await authDataset({
datasetId,
per: ReadPermissionVal,
req,
authToken: true,
authApiKey: true
});
const result = keys
.filter((key) => isS3ObjectKey(key, 'temp') && key.startsWith(`temp/${teamId}/`))
.map((key) => ({
key,
previewUrl: jwtSignS3DownloadToken({
objectKey: key,
bucketName: S3Buckets.private,
expiredTime: addHours(new Date(), 1)
})
}));
return GetSearchTestImagePreviewUrlsResponseSchema.parse(result);
}
export default NextAPI(handler);
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { authFrequencyLimit } from '@fastgpt/service/common/system/frequencyLimit/utils';
import { addSeconds } from 'date-fns';
import { getTeamPlanStatus } from '@fastgpt/service/support/wallet/sub/utils';
import { imageFileType } from '@fastgpt/global/common/file/constants';
import { parseAllowedExtensions } from '@fastgpt/service/common/s3/utils/uploadConstraints';
import { getFileS3Key } from '@fastgpt/service/common/s3/utils';
import { S3PrivateBucket } from '@fastgpt/service/common/s3/buckets/private';
import {
PresignSearchTestImageBodySchema,
PresignSearchTestImageResponseSchema,
type PresignSearchTestImageBody,
type PresignSearchTestImageResponse
} from '@fastgpt/global/openapi/core/dataset/file/api';
async function handler(
req: ApiRequestProps<PresignSearchTestImageBody>
): Promise<PresignSearchTestImageResponse> {
const { datasetId, filename } = PresignSearchTestImageBodySchema.parse(req.body);
const { teamId, userId } = await authDataset({
datasetId,
per: ReadPermissionVal,
req,
authToken: true,
authApiKey: true
});
const planStatus = await getTeamPlanStatus({ teamId });
await authFrequencyLimit({
eventId: `${userId}-uploadfile`,
maxAmount: planStatus.standard?.maxUploadFileCount || global.feConfigs.uploadFileMaxAmount,
expiredTime: addSeconds(new Date(), 30),
num: 1
});
const bucket = new S3PrivateBucket();
const { fileKey } = getFileS3Key.temp({ teamId, filename });
const result = await bucket.createPresignedPutUrl(
{ rawKey: fileKey, filename },
{
expiredHours: 3,
maxFileSize: planStatus.standard?.maxUploadFileSize ?? global.feConfigs.uploadFileMaxSize,
uploadConstraints: {
allowedExtensions: parseAllowedExtensions(imageFileType)
}
}
);
return PresignSearchTestImageResponseSchema.parse(result);
}
export default NextAPI(handler);
import { insertData2Dataset } from '@/service/core/dataset/data/controller'; import { createDatasetData } from '@/service/core/dataset/data/data';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push'; import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
...@@ -91,7 +91,7 @@ export async function generateVector(): Promise<any> { ...@@ -91,7 +91,7 @@ export async function generateVector(): Promise<any> {
return { return {
data data
}; };
} catch (error) { } catch {
return { return {
error: true error: true
}; };
...@@ -238,7 +238,7 @@ const rebuildData = async ({ trainingData }: { trainingData: TrainingDataType }) ...@@ -238,7 +238,7 @@ const rebuildData = async ({ trainingData }: { trainingData: TrainingDataType })
} }
}) })
); );
} catch (error) {} } catch {}
// update vector, update dataset_data rebuilding status, delete data from training // update vector, update dataset_data rebuilding status, delete data from training
// 1. Insert new vector to dataset_data // 1. Insert new vector to dataset_data
...@@ -281,7 +281,7 @@ const rebuildData = async ({ trainingData }: { trainingData: TrainingDataType }) ...@@ -281,7 +281,7 @@ const rebuildData = async ({ trainingData }: { trainingData: TrainingDataType })
const insertData = async ({ trainingData }: { trainingData: TrainingDataType }) => { const insertData = async ({ trainingData }: { trainingData: TrainingDataType }) => {
return mongoSessionRun(async (session) => { return mongoSessionRun(async (session) => {
// insert new data to dataset // insert new data to dataset
const { tokens } = await insertData2Dataset({ const { tokens } = await createDatasetData({
teamId: trainingData.teamId, teamId: trainingData.teamId,
tmbId: trainingData.tmbId, tmbId: trainingData.tmbId,
datasetId: trainingData.datasetId, datasetId: trainingData.datasetId,
......
...@@ -6,7 +6,13 @@ import type { ...@@ -6,7 +6,13 @@ import type {
GetQuoteDataBody as GetQuoteDataProps, GetQuoteDataBody as GetQuoteDataProps,
GetQuoteDataResponse, GetQuoteDataResponse,
InsertDataBody, InsertDataBody,
UpdateDatasetDataBody CreateDatasetDataIndexBody,
DeleteDatasetDataIndexBody,
DeleteDatasetDataIndexResponse,
DatasetDataIndexResponse,
UpdateDatasetDataIndexBody,
UpdateDatasetDataBody,
UpdateDatasetDataResponse
} from '@fastgpt/global/openapi/core/dataset/data/api'; } from '@fastgpt/global/openapi/core/dataset/data/api';
export const getDatasetDataList = (data: GetDatasetDataListProps) => export const getDatasetDataList = (data: GetDatasetDataListProps) =>
...@@ -25,7 +31,13 @@ export const postInsertData2Dataset = (data: InsertDataBody) => ...@@ -25,7 +31,13 @@ export const postInsertData2Dataset = (data: InsertDataBody) =>
* update one datasetData by id * update one datasetData by id
*/ */
export const putDatasetDataById = (data: UpdateDatasetDataBody) => export const putDatasetDataById = (data: UpdateDatasetDataBody) =>
PUT('/core/dataset/data/update', data); PUT<UpdateDatasetDataResponse>('/core/dataset/data/update', data);
export const createDatasetDataIndex = (data: CreateDatasetDataIndexBody) =>
POST<DatasetDataIndexResponse>('/core/dataset/data/index/create', data);
export const updateDatasetDataIndex = (data: UpdateDatasetDataIndexBody) =>
POST<DatasetDataIndexResponse>('/core/dataset/data/index/update', data);
/** /**
* 删除一条知识库数据 * 删除一条知识库数据
...@@ -33,6 +45,9 @@ export const putDatasetDataById = (data: UpdateDatasetDataBody) => ...@@ -33,6 +45,9 @@ export const putDatasetDataById = (data: UpdateDatasetDataBody) =>
export const delOneDatasetDataById = (id: string) => export const delOneDatasetDataById = (id: string) =>
DELETE<string>(`/core/dataset/data/delete`, { id }); DELETE<string>(`/core/dataset/data/delete`, { id });
export const deleteDatasetDataIndex = (data: DeleteDatasetDataIndexBody) =>
POST<DeleteDatasetDataIndexResponse>('/core/dataset/data/index/delete', data);
// Get quote data // Get quote data
export const getQuoteData = (data: GetQuoteDataProps) => export const getQuoteData = (data: GetQuoteDataProps) =>
POST<GetQuoteDataResponse>(`/core/dataset/data/getQuoteData`, data); POST<GetQuoteDataResponse>(`/core/dataset/data/getQuoteData`, data);
import { POST } from '@/web/common/api/request'; import { POST } from '@/web/common/api/request';
import type { import type {
GetSearchTestImagePreviewUrlsBody,
GetSearchTestImagePreviewUrlsResponse,
GetPreviewChunksBody, GetPreviewChunksBody,
GetPreviewChunksResponse, GetPreviewChunksResponse,
PresignDatasetFilePostUrlBody PresignDatasetFilePostUrlBody,
PresignSearchTestImageBody,
PresignSearchTestImageResponse
} from '@fastgpt/global/openapi/core/dataset/file/api'; } from '@fastgpt/global/openapi/core/dataset/file/api';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type'; import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
...@@ -14,3 +18,12 @@ export const getPreviewChunks = (data: GetPreviewChunksBody) => ...@@ -14,3 +18,12 @@ export const getPreviewChunks = (data: GetPreviewChunksBody) =>
maxQuantity: 1, maxQuantity: 1,
timeout: 600000 timeout: 600000
}); });
export const getUploadSearchTestImagePresignedUrl = (data: PresignSearchTestImageBody) =>
POST<PresignSearchTestImageResponse>('/core/dataset/file/presignSearchTestImage', data);
export const postGetSearchTestImagePreviewUrls = (data: GetSearchTestImagePreviewUrlsBody) =>
POST<GetSearchTestImagePreviewUrlsResponse>(
'/core/dataset/file/getSearchTestImagePreviewUrls',
data
);
...@@ -10,6 +10,10 @@ export type SearchTestStoreItemType = { ...@@ -10,6 +10,10 @@ export type SearchTestStoreItemType = {
time: Date; time: Date;
duration: string; duration: string;
results: SearchDataResponseItemType[]; results: SearchDataResponseItemType[];
queryImageRefs?: {
key: string;
previewUrl?: string;
}[];
searchMode: `${DatasetSearchModeEnum}`; searchMode: `${DatasetSearchModeEnum}`;
limit: number; limit: number;
usingReRank: boolean; usingReRank: boolean;
...@@ -27,7 +31,7 @@ type State = { ...@@ -27,7 +31,7 @@ type State = {
export const useSearchTestStore = create<State>()( export const useSearchTestStore = create<State>()(
devtools( devtools(
persist( persist(
immer((set, get) => ({ immer((set) => ({
datasetTestList: [], datasetTestList: [],
pushDatasetTestItem(data) { pushDatasetTestItem(data) {
set((state) => { set((state) => {
......
import { describe, expect, it } from 'vitest';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getRootUser } from '@test/datas/users';
import { hasSameValue } from '@/service/core/dataset/data/utils';
const createDatasetContext = async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test dataset',
teamId: root.teamId,
tmbId: root.tmbId,
type: DatasetTypeEnum.dataset,
vectorModel: 'text-embedding-3-small',
agentModel: 'gpt-4o-mini'
});
const collection = await MongoDatasetCollection.create({
name: 'test collection',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
return { root, dataset, collection };
};
describe('hasSameValue', () => {
it('should resolve when no data has identical q and a in the same collection', async () => {
const { root, dataset, collection } = await createDatasetContext();
await MongoDatasetData.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
q: 'same question',
a: 'old answer',
indexes: []
});
await expect(
hasSameValue({
teamId: String(root.teamId),
datasetId: String(dataset._id),
collectionId: String(collection._id),
q: 'same question',
a: 'new answer'
})
).resolves.toBeUndefined();
});
it('should reject when identical q and a already exist', async () => {
const { root, dataset, collection } = await createDatasetContext();
await MongoDatasetData.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
q: 'same question',
a: '',
indexes: []
});
await expect(
hasSameValue({
teamId: String(root.teamId),
datasetId: String(dataset._id),
collectionId: String(collection._id),
q: 'same question'
})
).rejects.toBe('已经存在完全一致的数据');
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment