Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit ff77300f by YeYuheng Committed by GitHub
Options

feat: add image dataset UI (#6932) 

* feat: add image dataset ui

feat: address dataset image review comments

fix: restore dataset test panel styles

refactor: merge dataset data index update api

perf: data controler code

perf: review

* refactor: improve dataset detail input panels

* test: add dataset data service coverage

* fix: avoid dataset index update race

---------

Co-authored-by: archer <545436317@qq.com>
parent 45be8908
Showing with 6021 additions and 2023 deletions
document/content/self-host/upgrading/4-15/41502.mdx
......@@ -11,6 +11,8 @@ description: 'FastGPT V4.15.0-beta2 更新说明'
2. 增加 workflow zod 检查鲁棒性。
3. 禁用工作流无效连接模式。
4. 增加工作流节点，名字超长适配。
5. 知识库搜索测试交互。
6. 知识库数据编辑弹窗。
## 🐛 修复
......
packages/global/core/ai/model.schema.ts
......@@ -100,6 +100,7 @@ export const EmbeddingModelItemSchema = PriceTypeSchema.extend(BaseModelItemSche
maxToken: z.number(), // model max token
weight: z.number(), // training weight
hidden: z.boolean().optional(), // Disallow creation
vision: z.boolean().optional(), // Support image embedding
normalization: z.boolean().optional(), // normalization processing
batchSize: z.number().optional(), // batch request size
defaultConfig: z.record(z.string(), z.any()).optional(), // post request config
......
packages/global/core/dataset/constants.ts
......@@ -13,7 +13,7 @@ export enum DatasetTypeEnum {
dingtalk = 'dingtalk'
}
// @ts-ignore
// @ts-expect-error ignore incomplete api dataset map
export const ApiDatasetTypeMap: Record<
`${DatasetTypeEnum}`,
{
......
packages/global/core/dataset/data/constants.ts
import { i18nT } from '../../../../web/i18n/utils';
export enum DatasetDataIndexTypeEnum {
default = 'default',
custom = 'custom',
summary = 'summary',
question = 'question',
image = 'image'
default = 'default', // 默认的
summary = 'summary', // 摘要，系统生成
question = 'question', // 补全问题，系统生成
image = 'image', // 图片描述，系统生成
custom = 'custom'
}
export const DatasetDataIndexMap: Record<
......
packages/global/core/dataset/type.ts
......@@ -337,6 +337,7 @@ export const DatasetDataItemSchema = DatasetDataFieldSchema.extend({
sourceId: z.string().optional().meta({ description: '来源 ID' }),
chunkIndex: z.number().meta({ description: '块索引' }),
indexes: z.array(DatasetDataIndexItemSchema).meta({ description: '向量索引' }),
imageDescMap: z.record(z.string(), z.string()).optional().meta({ description: '图片描述映射' }),
isOwner: z.boolean().meta({ description: '是否为 owner' })
});
export type DatasetDataItemType = z.infer<typeof DatasetDataItemSchema>;
......
packages/global/core/dataset/utils.ts
......@@ -40,7 +40,7 @@ export function getSourceNameIcon({
if (strIsLink(sourceId)) {
return 'common/linkBlue';
}
} catch (error) {}
} catch {}
return 'file/fill/file';
}
......@@ -51,3 +51,5 @@ export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => {
if (mode === TrainingModeEnum.image) return data.length * 2;
return data.length;
};
export const isDatasetFileObjectKey = (url: string) => /^(temp|chat|dataset)\//i.test(url);
packages/global/core/workflow/runtime/type.ts
......@@ -215,6 +215,17 @@ export const DispatchNodeResponseSchema = z
limit: z.number().optional().meta({ description: '限制' }),
searchMode: z.enum(DatasetSearchModeEnum).optional().meta({ description: '搜索模式' }),
embeddingWeight: z.number().optional().meta({ description: '嵌入权重' }),
filteredFileCount: z.number().optional().meta({ description: '过滤的非图片文件数量' }),
queryImages: z
.array(
z.object({
key: z.string().optional(),
url: z.string().optional(),
name: z.string().optional()
})
)
.optional()
.meta({ description: '参与知识库检索的图片' }),
rerankModel: z.string().optional().meta({ description: '重排模型' }),
rerankWeight: z.number().optional().meta({ description: '重排权重' }),
reRankInputTokens: z.number().optional().meta({ description: '重排输入 token' }),
......@@ -359,7 +370,10 @@ export type DispatchNodeResponseType = Omit<
toolDetail?: DispatchNodeResponseType[];
};
export type DispatchNodeResultType<T = {}, ERR = { [NodeOutputKeyEnum.errorText]?: string }> = {
export type DispatchNodeResultType<
T = unknown,
ERR = { [NodeOutputKeyEnum.errorText]?: string }
> = {
[DispatchNodeResponseKeyEnum.answerText]?: string;
[DispatchNodeResponseKeyEnum.reasoningText]?: string;
[DispatchNodeResponseKeyEnum.skipHandleId]?: string[]; // skip some edge handle id
......
packages/global/openapi/core/dataset/api.ts
......@@ -315,62 +315,76 @@ export type CreateDatasetFolderBody = z.infer<typeof CreateDatasetFolderBodySche
* API: 搜索测试
* Route: POST /api/core/dataset/searchTest
* ============================================================================ */
export const SearchDatasetTestBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
text: z.string().meta({
example: 'FastGPT 是什么',
description: '搜索文本'
}),
similarity: z.number().optional().meta({
example: 0.3,
description: '最低相似度阈值'
}),
limit: z.number().optional().meta({
example: 5000,
description: '最大返回 token 数'
}),
searchMode: z.enum(DatasetSearchModeEnum).optional().meta({
example: DatasetSearchModeEnum.mixedRecall,
description: '搜索模式'
}),
embeddingWeight: z.number().optional().meta({
example: 1,
description: '向量搜索权重'
}),
usingReRank: z.boolean().optional().meta({
description: '是否使用重排序'
}),
rerankModel: z.string().optional().meta({
description: '重排序模型名称'
}),
rerankWeight: z.number().optional().meta({
description: '重排序权重'
}),
datasetSearchUsingExtensionQuery: z.boolean().optional().meta({
description: '是否使用问题扩展'
}),
datasetSearchExtensionModel: z.string().optional().meta({
description: '问题扩展模型'
}),
datasetSearchExtensionBg: z.string().optional().meta({
description: '问题扩展背景描述'
}),
datasetDeepSearch: z.boolean().optional().meta({
description: '是否启用深度搜索'
}),
datasetDeepSearchModel: z.string().optional().meta({
description: '深度搜索模型'
}),
datasetDeepSearchMaxTimes: z.number().optional().meta({
description: '深度搜索最大轮次'
}),
datasetDeepSearchBg: z.string().optional().meta({
description: '深度搜索背景描述'
export const SearchDatasetTestBodySchema = z
.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
text: z.string().optional().default('').meta({
example: 'FastGPT 是什么',
description: '搜索文本'
}),
queryImageUrls: z
.array(z.string().min(1))
.max(10, '最多支持上传10张图片')
.optional()
.default([])
.meta({
example: ['temp/teamId/search-image.png'],
description:
'搜索测试图片临时 key，最多 10 张。需先调用 /api/core/dataset/file/presignSearchTestImage 获取预签名上传 URL 和 temp/${teamId}/... key，不支持直接传公网 URL、dataset key 或 chat key'
}),
similarity: z.number().optional().meta({
example: 0.3,
description: '最低相似度阈值'
}),
limit: z.number().optional().meta({
example: 5000,
description: '最大返回 token 数'
}),
searchMode: z.enum(DatasetSearchModeEnum).optional().meta({
example: DatasetSearchModeEnum.mixedRecall,
description: '搜索模式'
}),
embeddingWeight: z.number().optional().meta({
example: 1,
description: '向量搜索权重'
}),
usingReRank: z.boolean().optional().meta({
description: '是否使用重排序'
}),
rerankModel: z.string().optional().meta({
description: '重排序模型名称'
}),
rerankWeight: z.number().optional().meta({
description: '重排序权重'
}),
datasetSearchUsingExtensionQuery: z.boolean().optional().meta({
description: '是否使用问题扩展'
}),
datasetSearchExtensionModel: z.string().optional().meta({
description: '问题扩展模型'
}),
datasetSearchExtensionBg: z.string().optional().meta({
description: '问题扩展背景描述'
}),
datasetDeepSearch: z.boolean().optional().meta({
description: '是否启用深度搜索'
}),
datasetDeepSearchModel: z.string().optional().meta({
description: '深度搜索模型'
}),
datasetDeepSearchMaxTimes: z.number().optional().meta({
description: '深度搜索最大轮次'
}),
datasetDeepSearchBg: z.string().optional().meta({
description: '深度搜索背景描述'
})
})
});
.refine((data) => !!data.text.trim() || data.queryImageUrls.length > 0, {
message: 'text or queryImageUrls is required'
});
export type SearchDatasetTestBody = z.infer<typeof SearchDatasetTestBodySchema>;
export const SearchDatasetTestResponseSchema = z.object({
......
packages/global/openapi/core/dataset/data/api.ts
......@@ -53,6 +53,50 @@ export type GetDatasetDataDetailResponse = z.infer<typeof GetDatasetDataDetailRe
* ============================================================================ */
export const UpdateDatasetDataBodySchema = UpdateDatasetDataPropsSchema;
export type UpdateDatasetDataBody = z.infer<typeof UpdateDatasetDataBodySchema>;
export const UpdateDatasetDataResponseSchema = z.object({});
export type UpdateDatasetDataResponse = z.infer<typeof UpdateDatasetDataResponseSchema>;
const DatasetDataIndexContentSchema = z.object({
type: DatasetDataIndexItemSchema.shape.type.meta({
description: '索引类型'
}),
text: z.string().min(1).meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '索引文本'
})
});
/* ============================================================================
* API: 新增数据索引
* Route: POST /api/core/dataset/data/index/create
* ============================================================================ */
export const CreateDatasetDataIndexBodySchema = DatasetDataIndexContentSchema.extend({
dataId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
})
});
export type CreateDatasetDataIndexBody = z.infer<typeof CreateDatasetDataIndexBodySchema>;
export const DatasetDataIndexResponseSchema = z.object({
index: DatasetDataIndexItemSchema
});
export type DatasetDataIndexResponse = z.infer<typeof DatasetDataIndexResponseSchema>;
/* ============================================================================
* API: 更新数据索引
* Route: POST /api/core/dataset/data/index/update
* ============================================================================ */
export const UpdateDatasetDataIndexBodySchema = DatasetDataIndexContentSchema.extend({
dataId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
}),
indexDataId: z.string().meta({
example: '68ad85a7463006c963799a09',
description: '索引对应的 vectorDB ID'
})
});
export type UpdateDatasetDataIndexBody = z.infer<typeof UpdateDatasetDataIndexBodySchema>;
/* ============================================================================
* API: 删除数据集数据
......@@ -66,6 +110,28 @@ export const DeleteDatasetDataQuerySchema = z.object({
});
export type DeleteDatasetDataQuery = z.infer<typeof DeleteDatasetDataQuerySchema>;
export const DeleteDatasetDataResponseSchema = z.literal('success');
export type DeleteDatasetDataResponse = z.infer<typeof DeleteDatasetDataResponseSchema>;
/* ============================================================================
* API: 删除数据索引
* Route: POST /api/core/dataset/data/index/delete
* ============================================================================ */
export const DeleteDatasetDataIndexBodySchema = z.object({
dataId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
}),
indexDataId: z.string().meta({
example: '68ad85a7463006c963799a09',
description: '索引对应的 vectorDB ID'
})
});
export type DeleteDatasetDataIndexBody = z.infer<typeof DeleteDatasetDataIndexBodySchema>;
export const DeleteDatasetDataIndexResponseSchema = z.object({});
export type DeleteDatasetDataIndexResponse = z.infer<typeof DeleteDatasetDataIndexResponseSchema>;
/* ============================================================================
* API: 获取引用数据
* Route: POST /api/core/dataset/data/getQuoteData
......
packages/global/openapi/core/dataset/data/index.ts
......@@ -9,7 +9,15 @@ import {
InsertImagesBodySchema,
PushDataBodySchema,
GetDatasetDataListBodySchema,
GetDatasetDataListResponseSchema
GetDatasetDataListResponseSchema,
UpdateDatasetDataResponseSchema,
CreateDatasetDataIndexBodySchema,
DatasetDataIndexResponseSchema,
UpdateDatasetDataIndexBodySchema,
DeleteDatasetDataIndexBodySchema,
DeleteDatasetDataIndexResponseSchema,
GetDatasetDataDetailResponseSchema,
DeleteDatasetDataResponseSchema
} from './api';
export const DatasetDataPath: OpenAPIPath = {
......@@ -47,7 +55,12 @@ export const DatasetDataPath: OpenAPIPath = {
},
responses: {
200: {
description: '成功返回数据详情'
description: '成功返回数据详情',
content: {
'application/json': {
schema: GetDatasetDataDetailResponseSchema
}
}
}
}
}
......@@ -67,7 +80,87 @@ export const DatasetDataPath: OpenAPIPath = {
},
responses: {
200: {
description: '更新成功'
description: '更新成功',
content: {
'application/json': {
schema: UpdateDatasetDataResponseSchema
}
}
}
}
}
},
'/core/dataset/data/index/create': {
post: {
summary: '新增数据索引',
description: '新增指定数据下的单个自定义索引，不覆盖其它索引',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: CreateDatasetDataIndexBodySchema
}
}
},
responses: {
200: {
description: '保存成功',
content: {
'application/json': {
schema: DatasetDataIndexResponseSchema
}
}
}
}
}
},
'/core/dataset/data/index/update': {
post: {
summary: '更新数据索引',
description: '更新指定数据下的单个自定义索引，不覆盖其它索引',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: UpdateDatasetDataIndexBodySchema
}
}
},
responses: {
200: {
description: '更新成功',
content: {
'application/json': {
schema: DatasetDataIndexResponseSchema
}
}
}
}
}
},
'/core/dataset/data/index/delete': {
post: {
summary: '删除数据索引',
description: '删除指定数据下的单个索引，不触发整条数据重建',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: DeleteDatasetDataIndexBodySchema
}
}
},
responses: {
200: {
description: '删除成功',
content: {
'application/json': {
schema: DeleteDatasetDataIndexResponseSchema
}
}
}
}
}
......@@ -83,7 +176,12 @@ export const DatasetDataPath: OpenAPIPath = {
},
responses: {
200: {
description: '删除成功'
description: '删除成功',
content: {
'application/json': {
schema: DeleteDatasetDataResponseSchema
}
}
}
}
}
......
packages/global/openapi/core/dataset/file/api.ts
......@@ -2,6 +2,7 @@ import { z } from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo';
import { DatasetSourceReadTypeEnum } from '../../../../core/dataset/constants';
import { ChunkSettingsSchema } from '../../../../core/dataset/type';
import { CreatePostPresignedUrlResponseSchema } from '../../../../common/file/s3/type';
/* ============================================================================
* API: 预览文件分块
......@@ -68,3 +69,73 @@ export const PresignDatasetFilePostUrlBodySchema = z.object({
})
});
export type PresignDatasetFilePostUrlBody = z.infer<typeof PresignDatasetFilePostUrlBodySchema>;
export const PresignDatasetFilePostUrlResponseSchema = CreatePostPresignedUrlResponseSchema.meta({
description: 'S3 预签名上传 URL 及相关头信息'
});
export type PresignDatasetFilePostUrlResponse = z.infer<
typeof PresignDatasetFilePostUrlResponseSchema
>;
/* ============================================================================
* API: 获取搜索测试图片上传预签名 URL
* Route: POST /api/core/dataset/file/presignSearchTestImage
* Method: POST
* Description: 获取用于知识库搜索测试图片上传的临时预签名 URL，上传对象 3 小时后过期
* Tags: ['Dataset', 'File', 'Write']
* ============================================================================ */
export const PresignSearchTestImageBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
filename: z.string().min(1).meta({
example: 'demo.png',
description: '待上传图片文件名'
})
});
export type PresignSearchTestImageBody = z.infer<typeof PresignSearchTestImageBodySchema>;
export const PresignSearchTestImageResponseSchema = CreatePostPresignedUrlResponseSchema.meta({
description: '搜索测试图片上传预签名 URL、临时 key 和预览 URL'
});
export type PresignSearchTestImageResponse = z.infer<typeof PresignSearchTestImageResponseSchema>;
/* ============================================================================
* API: 获取搜索测试图片预览 URL
* Route: POST /api/core/dataset/file/getSearchTestImagePreviewUrls
* Method: POST
* Description: 根据搜索测试历史中的临时图片 key 重新生成短期预览 URL
* Tags: ['Dataset', 'File', 'Read']
* ============================================================================ */
export const GetSearchTestImagePreviewUrlsBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
keys: z
.array(z.string().min(1))
.max(10)
.meta({
example: ['temp/teamId/demo.png'],
description: '搜索测试图片临时 S3 key 列表，最多 10 个'
})
});
export type GetSearchTestImagePreviewUrlsBody = z.infer<
typeof GetSearchTestImagePreviewUrlsBodySchema
>;
export const GetSearchTestImagePreviewUrlsResponseSchema = z.array(
z.object({
key: z.string().meta({
example: 'temp/teamId/demo.png',
description: '临时图片 S3 key'
}),
previewUrl: z.string().meta({
description: '用于前端缩略图展示的临时预览 URL'
})
})
);
export type GetSearchTestImagePreviewUrlsResponse = z.infer<
typeof GetSearchTestImagePreviewUrlsResponseSchema
>;
packages/global/openapi/core/dataset/file/index.ts
import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag';
import {
GetSearchTestImagePreviewUrlsBodySchema,
GetSearchTestImagePreviewUrlsResponseSchema,
GetPreviewChunksBodySchema,
GetPreviewChunksResponseSchema,
PresignDatasetFilePostUrlBodySchema
PresignDatasetFilePostUrlBodySchema,
PresignDatasetFilePostUrlResponseSchema,
PresignSearchTestImageBodySchema,
PresignSearchTestImageResponseSchema
} from './api';
import { CreatePostPresignedUrlResponseSchema } from '../../../../common/file/s3/type';
export const DatasetFilePath: OpenAPIPath = {
'/core/dataset/file/getPreviewChunks': {
......@@ -49,7 +53,55 @@ export const DatasetFilePath: OpenAPIPath = {
description: '成功返回预签名上传 URL、key、请求头和最大文件大小',
content: {
'application/json': {
schema: CreatePostPresignedUrlResponseSchema
schema: PresignDatasetFilePostUrlResponseSchema
}
}
}
}
}
},
'/core/dataset/file/presignSearchTestImage': {
post: {
summary: '获取搜索测试图片上传预签名 URL',
description: '获取搜索测试图片上传预签名 URL，仅支持图片文件，上传对象 3 小时后过期',
tags: [TagsMap.datasetFile],
requestBody: {
content: {
'application/json': {
schema: PresignSearchTestImageBodySchema
}
}
},
responses: {
200: {
description: '成功返回临时图片上传 URL、key 和缩略图预览 URL',
content: {
'application/json': {
schema: PresignSearchTestImageResponseSchema
}
}
}
}
}
},
'/core/dataset/file/getSearchTestImagePreviewUrls': {
post: {
summary: '获取搜索测试图片预览 URL',
description: '根据搜索测试历史中的临时图片 key 重新生成短期预览 URL',
tags: [TagsMap.datasetFile],
requestBody: {
content: {
'application/json': {
schema: GetSearchTestImagePreviewUrlsBodySchema
}
}
},
responses: {
200: {
description: '成功返回临时图片 key 和缩略图预览 URL 列表',
content: {
'application/json': {
schema: GetSearchTestImagePreviewUrlsResponseSchema
}
}
}
......
packages/global/openapi/core/dataset/index.ts
......@@ -183,7 +183,8 @@ export const DatasetPath: OpenAPIPath = {
'/core/dataset/searchTest': {
post: {
summary: '搜索测试',
description: '对知识库执行搜索测试,支持多种搜索模式、重排序和问题扩展',
description:
'对知识库执行搜索测试，支持多种搜索模式、重排序、问题扩展和临时图片 key 检索。图片检索需先调用 /core/dataset/file/presignSearchTestImage 获取预签名上传 URL 和 temp/${teamId}/... key',
tags: [TagsMap.datasetCommon],
requestBody: {
content: {
......
packages/service/common/mongo/init.ts
......@@ -5,7 +5,7 @@ import { serviceEnv } from '../../env';
const logger = getLogger(LogCategories.INFRA.MONGO);
const maxConnecting = Math.max(30, serviceEnv.DB_MAX_LINK);
const maxConnecting = Math.max(5, serviceEnv.DB_MAX_LINK);
/**
* connect MongoDB and init data
......@@ -50,7 +50,7 @@ export async function connectMongo(props: {
bufferCommands: true,
maxConnecting: maxConnecting, // 最大连接数: 防止连接数过多时无法满足需求
maxPoolSize: maxConnecting, // 最大连接池大小: 防止连接池过大时无法满足需求
minPoolSize: 20, // 最小连接数: 20,防止连接数过少时无法满足需求
minPoolSize: 1,
connectTimeoutMS: 60000, // 连接超时: 60秒,防止连接失败时长时间阻塞
waitQueueTimeoutMS: 60000, // 等待队列超时: 60秒,防止等待队列长时间阻塞
socketTimeoutMS: 60000, // Socket 超时: 60秒,防止Socket连接失败时长时间阻塞
......
packages/web/i18n/en/account.json
......@@ -95,6 +95,7 @@
"model.vision": "Vision model",
"model.vision_tag": "Vision",
"model.vision_tip": "If the model supports image recognition, turn on this switch.",
"model.embedding_vision_tip": "Enable this when the embedding model can accept image input for image vector indexing and image search.",
"model.voices": "voice role",
"model.voices_tip": "Configure multiple through an array, for example:\n\n[\n {\n \"label\": \"Alloy\",\n \"value\": \"alloy\"\n },\n {\n \"label\": \"Echo\",\n \"value\": \"echo\"\n }\n]",
"model_provider": "Model Provider",
......
packages/web/i18n/en/account_model.json
......@@ -81,7 +81,7 @@
"view_chart": "Chart",
"view_table": "Table",
"vlm_model": "Vlm",
"vlm_model_tip": "Used to generate additional indexing of images in a document in the knowledge base",
"vlm_model_tip": "Automatically labels images in documents and generates text descriptions to assist text retrieval",
"volunme_of_failed_calls": "Error amount",
"waiting_test": "Waiting for testing"
}
packages/web/i18n/en/common.json
......@@ -240,6 +240,8 @@
"core.ai.Prompt": "Prompt",
"core.ai.Support tool": "Tool call",
"core.ai.model.Dataset Agent Model": "File read model",
"core.ai.model.multimodal": "Multimodal",
"core.ai.model.multimodal_tip": "Multimodal embedding models can generate vectors for images.",
"core.ai.model.Vector Model": "Index model",
"core.ai.model.doc_index_and_dialog": "Document Index & Dialog Index",
"core.app.Api request": "API Request",
......@@ -458,7 +460,7 @@
"core.dataset.data.Search data placeholder": "Search Related Data",
"core.dataset.data.Updated": "Updated",
"core.dataset.data.group": " Groups",
"core.dataset.embedding model tip": "The index model can convert natural language into vectors for semantic search.\nNote that different index models cannot be used together. Once an index model is selected, it cannot be changed.",
"core.dataset.embedding model tip": "The index model converts knowledge base content into vectors for semantic search. Note that knowledge bases using different index models cannot be queried together. Switching the index model requires rebuilding all vector indexes, so choose carefully.",
"core.dataset.error.Data not found": "Data Not Found or Deleted",
"core.dataset.error.Start Sync Failed": "Failed to Start Sync",
"core.dataset.error.unExistDataset": "The knowledge base does not exist",
......@@ -522,7 +524,14 @@
"core.dataset.test.Test Result": "Test Result",
"core.dataset.test.Test Text": "Single Text Test",
"core.dataset.test.Test Text Placeholder": "Enter the text to be tested",
"core.dataset.test.image_expired": "Image expired",
"core.dataset.test.image_search_disabled_tip": "Configure an image understanding model or multimodal embedding model first.",
"core.dataset.test.image_token": "[Image]",
"core.dataset.test.input_title": "Test input",
"core.dataset.test.max_images_tip": "Up to 10 images are supported",
"core.dataset.test.search_config": "Search configuration",
"core.dataset.test.Test params": "Test Parameters",
"core.dataset.test.upload_image": "Upload image",
"core.dataset.test.delete test history": "Delete This Test Result",
"core.dataset.test.test history": "Test History",
"core.dataset.test.test result placeholder": "Test results will be displayed here",
......@@ -727,6 +736,7 @@
"dataset.data.Index Placeholder": "Enter Index Text Content",
"dataset.data.Input Success Tip": "Data Imported Successfully",
"dataset.data.Update Success Tip": "Data Updated Successfully",
"dataset.data.Update Index Timeout Tip": "Updating indexes timed out. Please refresh the data later to check the result.",
"dataset.data.edit.Index": "Data Index ({{amount}})",
"dataset.dataset_name": "Dataset Name",
"dataset.deleteFolderTips": "Confirm to Delete This Folder and All Its Contained Datasets? Data Cannot Be Recovered After Deletion, Please Confirm!",
......
packages/web/i18n/en/dataset.json
......@@ -36,6 +36,8 @@
"common.error.unKnow": "Unknown error",
"common_dataset": "General Dataset",
"common_dataset_desc": "Building a knowledge base by importing files, web page links, or manual entry",
"create_dataset_title": "Create {{name}}",
"dataset_name_placeholder": "Give the knowledge base a name",
"confirm_delete_collection": "Confirm to delete {{num }} files?",
"confirm_import_images": "Total {{num}} | Confirm create",
"confirm_to_rebuild_embedding_tip": "This will re-vectorize all data in the knowledge base. It may take a while depending on the data volume. Proceed?",
......@@ -86,8 +88,14 @@
"file_model_function_tip": "Used for QA generation, auto-indexing, and other AI-powered data processing.",
"filename": "Filename",
"folder_dataset": "Folder",
"generate_index": "Update index",
"image_auto_parse": "Automatic image indexing",
"image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes",
"image_auto_parse_tip_commercial": "Upgrade to the commercial edition to use this feature",
"image_auto_parse_tip_multimodal_with_vlm": "Generate image vector indexes and text description indexes for document images to support image search",
"image_auto_parse_tip_multimodal_without_vlm": "Use a multimodal model to generate image vector indexes and support image search",
"image_auto_parse_tip_vlm_only": "Use VLM to automatically label document images and generate text description indexes",
"image_auto_parse_tip_no_vlm_or_multimodal": "Configure an image understanding model or switch to a multimodal vector model before enabling this",
"images_creating": "Creating",
"immediate_sync": "Immediate Synchronization",
"import_confirm": "Start import",
......@@ -186,6 +194,7 @@
"uploading_progress": "Uploading: {{num}}%",
"vector_model_max_tokens_tip": "Each chunk of data has a maximum length of 3000 tokens",
"vllm_model": "Image understanding model",
"vllm_model_tip": "Automatically labels images in documents and generates text descriptions to assist text retrieval",
"website_dataset": "Web sync",
"website_dataset_desc": "Build knowledge base by crawling web page data in batches",
"website_info": "Website Information",
......
packages/web/i18n/en/file.json
{
"Image_Preview": "Picture preview",
"Image_dataset_requires_VLM_model_to_be_configured": "The image dataset needs to be configured with the image understanding model (VLM) to be used. Please add a model that supports image understanding in the model configuration first.",
"Image_dataset_requires_VLM_model_to_be_configured": "Image datasets require an image understanding model or a multimodal index model. Please configure a matching model first.",
"click_to_view_raw_source": "Click to View Original Source",
"file_name": "Filename",
"file_size": "Filesize",
......
packages/web/i18n/zh-CN/account.json
......@@ -95,6 +95,7 @@
"model.vision": "支持图片识别",
"model.vision_tag": "视觉",
"model.vision_tip": "如果模型支持图片识别，则打开该开关。",
"model.embedding_vision_tip": "开启后该索引模型可接收图片输入，并用于图片向量索引和图搜图。",
"model.voices": "声音角色",
"model.voices_tip": "通过一个数组配置多个，例如：\n[\n {\n \"label\": \"Alloy\",\n \"value\": \"alloy\"\n },\n {\n \"label\": \"Echo\",\n \"value\": \"echo\"\n }\n]",
"model_provider": "模型提供商",
......
packages/web/i18n/zh-CN/account_model.json
......@@ -81,7 +81,7 @@
"view_chart": "图表",
"view_table": "表格",
"vlm_model": "图片理解模型",
"vlm_model_tip": "用于知识库中对文档中的图片进行额外的索引生成",
"vlm_model_tip": "自动标注文档里的图片并生成文本描述，辅助文本检索",
"volunme_of_failed_calls": "调用失败量",
"waiting_test": "等待测试"
}
packages/web/i18n/zh-CN/common.json
......@@ -240,6 +240,8 @@
"core.ai.Prompt": "提示词",
"core.ai.Support tool": "工具调用",
"core.ai.model.Dataset Agent Model": "文本理解模型",
"core.ai.model.multimodal": "多模态",
"core.ai.model.multimodal_tip": "多模态索引模型可以给图片生成向量。",
"core.ai.model.Vector Model": "索引模型",
"core.ai.model.doc_index_and_dialog": "文档索引 & 对话索引",
"core.app.Api request": "API 访问",
......@@ -458,7 +460,7 @@
"core.dataset.data.Search data placeholder": "搜索相关数据",
"core.dataset.data.Updated": "已更新",
"core.dataset.data.group": "组",
"core.dataset.embedding model tip": "索引模型可以将自然语言转成向量，用于进行语义检索。\n注意，不同索引模型无法一起使用，选择完索引模型后将无法修改。",
"core.dataset.embedding model tip": "索引模型可以将知识库内容转成向量，用于进行语义检索。注意，不同索引模型的知识库无法同时查询，切换索引模型需重建全量向量索引，请慎重选择。",
"core.dataset.error.Data not found": "数据不存在或已被删除",
"core.dataset.error.Start Sync Failed": "开始同步失败",
"core.dataset.error.unExistDataset": "知识库不存在",
......@@ -521,8 +523,15 @@
"core.dataset.test.Test": "测试",
"core.dataset.test.Test Result": "测试结果",
"core.dataset.test.Test Text": "单个文本测试",
"core.dataset.test.Test Text Placeholder": "输入需要测试的文本",
"core.dataset.test.Test Text Placeholder": "输入需要测试的内容",
"core.dataset.test.image_expired": "图片已过期",
"core.dataset.test.image_search_disabled_tip": "请配置图片理解模型或多模态索引模型",
"core.dataset.test.image_token": "[图片]",
"core.dataset.test.input_title": "输入测试内容",
"core.dataset.test.max_images_tip": "最多支持上传10张图片",
"core.dataset.test.search_config": "搜索配置",
"core.dataset.test.Test params": "测试参数",
"core.dataset.test.upload_image": "上传图片",
"core.dataset.test.delete test history": "删除该测试结果",
"core.dataset.test.test history": "测试历史",
"core.dataset.test.test result placeholder": "测试结果将在这里展示",
......@@ -727,6 +736,7 @@
"dataset.data.Index Placeholder": "输入索引文本内容",
"dataset.data.Input Success Tip": "导入数据成功",
"dataset.data.Update Success Tip": "更新数据成功",
"dataset.data.Update Index Timeout Tip": "更新索引超时，请稍后刷新数据查看结果",
"dataset.data.edit.Index": "数据索引({{amount}})",
"dataset.dataset_name": "知识库名称",
"dataset.deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！",
......
packages/web/i18n/zh-CN/dataset.json
......@@ -36,6 +36,8 @@
"common.error.unKnow": "未知错误",
"common_dataset": "通用知识库",
"common_dataset_desc": "通过导入文件、网页链接或手动录入形式构建知识库",
"create_dataset_title": "创建{{name}}",
"dataset_name_placeholder": "给知识库取一个名字",
"confirm_delete_collection": "确认删除 {{num }} 个文件？",
"confirm_import_images": "共 {{num}} 张图片 | 确认创建",
"confirm_to_rebuild_embedding_tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。\n\n此外，你还需要注意修改选择该知识库的应用，避免它们与其他索引模型知识库混用。",
......@@ -86,8 +88,14 @@
"file_model_function_tip": "用于增强索引和 QA 生成",
"filename": "文件名",
"folder_dataset": "文件夹",
"generate_index": "更新索引",
"image_auto_parse": "图片自动索引",
"image_auto_parse_tips": "调用 VLM 自动标注文档里的图片，并生成额外的检索索引",
"image_auto_parse_tip_commercial": "请升级商业版后使用该功能",
"image_auto_parse_tip_multimodal_with_vlm": "为文档中的图片生成图片向量索引和文本描述索引，支持以图搜图",
"image_auto_parse_tip_multimodal_without_vlm": "使用多模态模型为图片生成向量索引，支持以图搜图",
"image_auto_parse_tip_vlm_only": "调用 VLM 自动标注文档里的图片，并生成文本描述索引",
"image_auto_parse_tip_no_vlm_or_multimodal": "需配置图片理解模型，或切换多模态向量模型后，方可启用",
"images_creating": "正在创建",
"immediate_sync": "立即同步",
"import_confirm": "确认上传",
......@@ -186,6 +194,7 @@
"uploading_progress": "上传中: {{num}}%",
"vector_model_max_tokens_tip": "每个分块数据，最大长度为 3000 tokens",
"vllm_model": "图片理解模型",
"vllm_model_tip": "自动标注文档里的图片并生成文本描述，辅助文本检索",
"website_dataset": "Web 站点同步",
"website_dataset_desc": "通过爬虫，批量爬取网页数据构建知识库",
"website_info": "网站信息",
......
packages/web/i18n/zh-CN/file.json
{
"Image_Preview": "图片预览",
"Image_dataset_requires_VLM_model_to_be_configured": "图片数据集需要配置图片理解模型(VLM)才能使用，请先在模型配置中添加支持图片理解的模型",
"Image_dataset_requires_VLM_model_to_be_configured": "图片数据集需要配置图片理解模型或多模态索引模型才能使用，请先在模型配置中添加对应模型",
"click_to_view_raw_source": "点击查看来源",
"file_name": "文件名",
"file_size": "文件大小",
......
packages/web/i18n/zh-Hant/account.json
......@@ -95,6 +95,7 @@
"model.vision": "支援圖片識別",
"model.vision_tag": "視覺",
"model.vision_tip": "如果模型支援圖片識別，則開啟該開關。",
"model.embedding_vision_tip": "開啟後該索引模型可接收圖片輸入，並用於圖片向量索引和圖搜圖。",
"model.voices": "聲音角色",
"model.voices_tip": "透過一個陣列設定多個，例如：\n[\n {\n \"label\": \"Alloy\",\n \"value\": \"alloy\"\n },\n {\n \"label\": \"Echo\",\n \"value\": \"echo\"\n }\n]",
"model_provider": "模型提供者",
......
packages/web/i18n/zh-Hant/account_model.json
......@@ -81,7 +81,7 @@
"view_chart": "圖表",
"view_table": "表格",
"vlm_model": "圖片理解模型",
"vlm_model_tip": "用於知識庫中對文件中的圖片進行額外的索引生成",
"vlm_model_tip": "自動標註文件裡的圖片並生成文字描述，輔助文字檢索",
"volunme_of_failed_calls": "調用失敗量",
"waiting_test": "等待測試"
}
packages/web/i18n/zh-Hant/common.json
......@@ -237,6 +237,8 @@
"core.ai.Prompt": "提示詞",
"core.ai.Support tool": "工具調用",
"core.ai.model.Dataset Agent Model": "檔案處理模型",
"core.ai.model.multimodal": "多模態",
"core.ai.model.multimodal_tip": "多模態索引模型可以給圖片生成向量。",
"core.ai.model.Vector Model": "索引模型",
"core.ai.model.doc_index_and_dialog": "文件索引與對話索引",
"core.app.Api request": "API 存取",
......@@ -453,7 +455,7 @@
"core.dataset.data.Search data placeholder": "搜尋相關資料",
"core.dataset.data.Updated": "已更新",
"core.dataset.data.group": "組",
"core.dataset.embedding model tip": "索引模型可以將自然語言轉換成向量，用於進行語意搜尋。\n注意，不同索引模型無法一起使用。選擇索引模型後就無法修改。",
"core.dataset.embedding model tip": "索引模型可以將知識庫內容轉成向量，用於進行語意檢索。注意，不同索引模型的知識庫無法同時查詢，切換索引模型需重建全量向量索引，請慎重選擇。",
"core.dataset.error.Data not found": "資料不存在或已被刪除",
"core.dataset.error.Start Sync Failed": "開始同步失敗",
"core.dataset.error.unExistDataset": "知識庫不存在",
......@@ -516,8 +518,15 @@
"core.dataset.test.Test": "測試",
"core.dataset.test.Test Result": "測試結果",
"core.dataset.test.Test Text": "單一文字測試",
"core.dataset.test.Test Text Placeholder": "輸入需要測試的文字",
"core.dataset.test.Test Text Placeholder": "輸入需要測試的內容",
"core.dataset.test.image_expired": "圖片已過期",
"core.dataset.test.image_search_disabled_tip": "請配置圖片理解模型或多模態索引模型",
"core.dataset.test.image_token": "[圖片]",
"core.dataset.test.input_title": "輸入測試內容",
"core.dataset.test.max_images_tip": "最多支援上傳10張圖片",
"core.dataset.test.search_config": "搜尋設定",
"core.dataset.test.Test params": "測試參數",
"core.dataset.test.upload_image": "上傳圖片",
"core.dataset.test.delete test history": "刪除此測試結果",
"core.dataset.test.test history": "測試歷史",
"core.dataset.test.test result placeholder": "測試結果將顯示在這裡",
......@@ -722,6 +731,7 @@
"dataset.data.Index Placeholder": "輸入索引文字內容",
"dataset.data.Input Success Tip": "匯入資料成功",
"dataset.data.Update Success Tip": "更新資料成功",
"dataset.data.Update Index Timeout Tip": "更新索引逾時，請稍後重新整理資料查看結果",
"dataset.data.edit.Index": "資料索引（{{amount}}）",
"dataset.dataset_name": "知識庫名稱",
"dataset.deleteFolderTips": "確認刪除此資料夾及其包含的所有知識庫？刪除後資料無法復原，請確認！",
......
packages/web/i18n/zh-Hant/dataset.json
......@@ -36,6 +36,8 @@
"common.error.unKnow": "未知錯誤",
"common_dataset": "通用資料集",
"common_dataset_desc": "通過導入文件、網頁鏈接或手動錄入形式構建知識庫",
"create_dataset_title": "建立{{name}}",
"dataset_name_placeholder": "給知識庫取一個名字",
"confirm_delete_collection": "確認刪除 {{num }} 個文件？",
"confirm_import_images": "共 {{num}} 張圖片 | 確認創建",
"confirm_to_rebuild_embedding_tip": "確定要為資料集切換索引嗎？\n切換索引是一個重要的操作，需要對您資料集內所有資料重新建立索引，可能需要較長時間，請確保帳號內剩餘點數充足。\n\n此外，您還需要注意修改使用此資料集的應用程式，避免與其他索引模型資料集混用。",
......@@ -86,8 +88,14 @@
"file_model_function_tip": "用於增強索引和問答生成",
"filename": "檔案名稱",
"folder_dataset": "資料夾",
"generate_index": "更新索引",
"image_auto_parse": "圖片自動索引",
"image_auto_parse_tips": "呼叫 VLM 自動標註文件裡的圖片，並生成額外的檢索索引",
"image_auto_parse_tip_commercial": "請升級商業版後使用該功能",
"image_auto_parse_tip_multimodal_with_vlm": "為文件中的圖片生成圖片向量索引和文字描述索引，支援以圖搜圖",
"image_auto_parse_tip_multimodal_without_vlm": "使用多模態模型為圖片生成向量索引，支援以圖搜圖",
"image_auto_parse_tip_vlm_only": "呼叫 VLM 自動標註文件裡的圖片，並生成文字描述索引",
"image_auto_parse_tip_no_vlm_or_multimodal": "需設定圖片理解模型，或切換多模態向量模型後，方可啟用",
"images_creating": "正在創建",
"immediate_sync": "立即同步",
"import_confirm": "確認上傳",
......@@ -186,6 +194,7 @@
"uploading_progress": "上傳中: {{num}}%",
"vector_model_max_tokens_tip": "每個分塊資料，最大長度為 3000 tokens",
"vllm_model": "圖片理解模型",
"vllm_model_tip": "自動標註文件裡的圖片並生成文字描述，輔助文字檢索",
"website_dataset": "網站同步",
"website_dataset_desc": "通過爬蟲，批量爬取網頁數據構建知識庫",
"website_info": "網站資訊",
......
packages/web/i18n/zh-Hant/file.json
{
"Image_Preview": "圖片預覽",
"Image_dataset_requires_VLM_model_to_be_configured": "圖片數據集需要配置圖片理解模型(VLM)才能使用，請先在模型配置中添加支持圖片理解的模型",
"Image_dataset_requires_VLM_model_to_be_configured": "圖片數據集需要設定圖片理解模型或多模態索引模型才能使用，請先在模型設定中新增對應模型",
"click_to_view_raw_source": "點選檢視原始來源",
"file_name": "檔案名稱",
"file_size": "檔案大小",
......
projects/app/src/components/Select/AIModelSelector.tsx
......@@ -8,9 +8,11 @@ import MySelect, { type SelectProps } from '@fastgpt/web/components/common/MySel
import MultipleRowSelect from '@fastgpt/web/components/common/MySelect/MultipleRowSelect';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import TestModeBetaTag from '@/components/core/ai/TestModeBetaTag';
import MultimodalTag from '@/components/core/ai/MultimodelTag';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next';
import React, { useCallback, useMemo, useState } from 'react';
import React, { useCallback, useMemo } from 'react';
import { ModelTypeEnum } from '@fastgpt/global/core/ai/constants';
type Props = SelectProps & {
disableTip?: string;
......@@ -18,14 +20,50 @@ type Props = SelectProps & {
cacheModel?: boolean;
};
const modelAvatarSizeMap = {
sm: '1rem',
md: '1.2rem',
lg: '1.4rem'
} as const;
const getModelAvatarSize = (size?: Props['size']) => {
if (typeof size === 'string' && size in modelAvatarSizeMap) {
return modelAvatarSizeMap[size as keyof typeof modelAvatarSizeMap];
}
return modelAvatarSizeMap.md;
};
const isTestModeModel = (model?: SystemModelItemType) => {
return !!model?.testMode;
};
const isMultimodalEmbeddingModel = (model?: SystemModelItemType) => {
return model?.type === ModelTypeEnum.embedding && !!model.vision;
};
const SelectorActiveModelTags = React.memo(function SelectorActiveModelTags({
model
}: {
model?: SystemModelItemType;
}) {
const showTestModeTip = isTestModeModel(model);
const showMultimodalTip = isMultimodalEmbeddingModel(model);
if (!showTestModeTip && !showMultimodalTip) return null;
const SelectorActiveTestModeTip = React.memo(function SelectorActiveTestModeTip() {
return (
<Box position={'absolute'} top={'50%'} right={'40px'} transform={'translateY(-50%)'} zIndex={3}>
<TestModeBetaTag />
<Box
position={'absolute'}
top={'50%'}
right={'40px'}
transform={'translateY(-50%)'}
zIndex={3}
display={'flex'}
alignItems={'center'}
gap={1}
>
{showTestModeTip && <TestModeBetaTag />}
{showMultimodalTip && <MultimodalTag />}
</Box>
);
});
......@@ -33,14 +71,16 @@ const SelectorActiveTestModeTip = React.memo(function SelectorActiveTestModeTip(
const ModelOptionLabel = React.memo(function ModelOptionLabel({
name,
showTestModeTip,
showMultimodalTip,
noOfLines
}: {
name: string;
showTestModeTip: boolean;
showMultimodalTip?: boolean;
noOfLines?: ResponsiveValue<number>;
}) {
return (
<Flex alignItems={'center'} flex={'1 1 0'} minW={0} overflow={'hidden'}>
<Flex alignItems={'center'} flex={'1 1 0'} w={'100%'} minW={0} overflow={'hidden'}>
<Box noOfLines={noOfLines ?? 1} flex={'1 1 0'} minW={0} overflow={'hidden'}>
{name}
</Box>
......@@ -49,6 +89,11 @@ const ModelOptionLabel = React.memo(function ModelOptionLabel({
<TestModeBetaTag />
</Box>
)}
{showMultimodalTip && (
<Box ml={1} flexShrink={0} pointerEvents={'auto'}>
<MultimodalTag />
</Box>
)}
</Flex>
);
});
......@@ -86,15 +131,7 @@ const OneRowSelector = ({
}
);
const avatarSize = useMemo(() => {
const size = {
sm: '1rem',
md: '1.2rem',
lg: '1.4rem'
};
//@ts-ignore
return props.size ? size[props.size] : size['md'];
}, [props.size]);
const avatarSize = useMemo(() => getModelAvatarSize(props.size), [props.size]);
const allModels = useMemo(
() => [
...llmModelList,
......@@ -123,7 +160,7 @@ const OneRowSelector = ({
return {
value: item.value,
label: (
<Flex alignItems={'center'} py={1} minW={0}>
<Flex alignItems={'center'} py={1} w={'100%'} minW={0}>
<Avatar
borderRadius={'0'}
mr={2}
......@@ -135,6 +172,7 @@ const OneRowSelector = ({
name={modelData.name}
noOfLines={noOfLines}
showTestModeTip={isTestModeModel(modelData)}
showMultimodalTip={isMultimodalEmbeddingModel(modelData)}
/>
</Flex>
)
......@@ -174,6 +212,7 @@ const OneRowSelector = ({
name={selectedModelData.name}
noOfLines={noOfLines}
showTestModeTip={false}
showMultimodalTip={false}
/>
</Flex>
) : undefined
......@@ -187,7 +226,7 @@ const OneRowSelector = ({
}}
/>
</MyTooltip>
{isTestModeModel(selectedModelData) && <SelectorActiveTestModeTip />}
<SelectorActiveModelTags model={selectedModelData} />
</Box>
);
};
......@@ -238,21 +277,15 @@ const MultipleRowSelector = ({
myModels
]);
const [value, setValue] = useState<string[]>([]);
const avatarSize = useMemo(() => {
const size = {
sm: '1rem',
md: '1.2rem',
lg: '1.4rem'
};
//@ts-ignore
return props.size ? size[props.size] : size['md'];
}, [props.size]);
const avatarSize = useMemo(() => getModelAvatarSize(props.size), [props.size]);
const selectedModelData = useMemo(
() => modelList.find((model) => model?.model === props.value),
[modelList, props.value]
);
const value = useMemo(
() => (selectedModelData ? [selectedModelData.provider, selectedModelData.model] : []),
[selectedModelData]
);
const selectorList = useMemo(() => {
const renderList = getModelProviders(i18n.language).map<{
......@@ -285,7 +318,13 @@ const MultipleRowSelector = ({
provider?.children.push({
label: (
<ModelOptionLabel name={modelData.name} showTestModeTip={isTestModeModel(modelData)} />
<Flex w={'100%'} minW={0}>
<ModelOptionLabel
name={modelData.name}
showTestModeTip={isTestModeModel(modelData)}
showMultimodalTip={isMultimodalEmbeddingModel(modelData)}
/>
</Flex>
),
value: modelData.model
});
......@@ -304,13 +343,9 @@ const MultipleRowSelector = ({
const SelectedLabel = useMemo(() => {
if (loading) return <>{t('common:model_loading')}</>;
if (!props.value) return <>{t('common:not_model_config')}</>;
const modelData = modelList.find((model) => model?.model === props.value);
if (!modelData) return <>{t('common:not_model_config')}</>;
if (!selectedModelData) return <>{t('common:not_model_config')}</>;
setValue([modelData.provider, props.value]);
const avatar = getModelProvider(modelData.provider)?.avatar;
const avatar = getModelProvider(selectedModelData.provider)?.avatar;
return (
<Flex alignItems={'center'} py={1} minW={0} overflow={'hidden'}>
......@@ -321,10 +356,15 @@ const MultipleRowSelector = ({
fallbackSrc={HUGGING_FACE_ICON}
w={avatarSize}
/>
<ModelOptionLabel name={modelData.name} noOfLines={noOfLines} showTestModeTip={false} />
<ModelOptionLabel
name={selectedModelData.name}
noOfLines={noOfLines}
showTestModeTip={false}
showMultimodalTip={false}
/>
</Flex>
);
}, [loading, props.value, t, modelList, getModelProvider, avatarSize, noOfLines]);
}, [loading, props.value, t, selectedModelData, getModelProvider, avatarSize, noOfLines]);
return (
<Box
......@@ -351,7 +391,7 @@ const MultipleRowSelector = ({
}}
/>
</MyTooltip>
{isTestModeModel(selectedModelData) && <SelectorActiveTestModeTip />}
<SelectorActiveModelTags model={selectedModelData} />
</Box>
);
};
......
projects/app/src/components/core/ai/MultimodelTag.tsx 0 → 100644
import { Box } from '@chakra-ui/react';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useTranslation } from 'next-i18next';
import React from 'react';
const MultimodalTag = React.memo(function MultimodalTag() {
const { t } = useTranslation();
return (
<MyTooltip label={t('common:core.ai.model.multimodal_tip')} shouldWrapChildren={false}>
<Box
display={'inline-flex'}
alignItems={'center'}
justifyContent={'center'}
flexShrink={0}
px={'8px'}
py={'4px'}
borderRadius={'6px'}
bg={'#F0FBFF'}
color={'#005B9C'}
fontSize={'10px'}
lineHeight={'14px'}
fontWeight={500}
whiteSpace={'nowrap'}
>
{t('common:core.ai.model.multimodal')}
</Box>
</MyTooltip>
);
});
export default MultimodalTag;
projects/app/src/components/core/chat/ChatContainer/ChatBox/components/SelectMarkCollection.tsx
......@@ -10,7 +10,9 @@ import { type AdminFbkType } from '@fastgpt/global/core/chat/type';
import SelectCollections from '@/web/core/dataset/components/SelectCollections';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
const InputDataModal = dynamic(() => import('@/pageComponents/dataset/detail/InputDataModal'));
const InputDataModal = dynamic(
() => import('@/pageComponents/dataset/detail/components/InputDataModal')
);
export type AdminMarkType = {
feedbackDataId?: string;
......
projects/app/src/components/core/chat/components/WholeResponseModal.tsx
/* eslint-disable react-hooks/static-components */
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Box, Flex, type BoxProps, useDisclosure, HStack, Grid } from '@chakra-ui/react';
import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type';
......@@ -19,6 +20,7 @@ import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import { completionFinishReasonMap } from '@fastgpt/global/core/ai/constants';
import { useSafeTranslation } from '@fastgpt/web/hooks/useSafeTranslation';
import dynamic from 'next/dynamic';
import ImagePreviewToken from '@/components/core/dataset/ImagePreviewToken';
const RequestIdDetailModal = dynamic(() => import('@/components/core/ai/requestId'), {
ssr: false
......@@ -35,6 +37,36 @@ type sideTabItemType = {
children: sideTabItemType[];
};
const QueryWithImages = React.memo(function QueryWithImages({
query,
queryImages,
datasetId
}: {
query?: string;
queryImages: NonNullable<ChatHistoryItemResType['queryImages']>;
datasetId?: string;
}) {
return (
<Box
border={'1px solid'}
borderColor={'myGray.200'}
borderRadius={'6px'}
bg={'myGray.50'}
color={'myGray.900'}
minH={'32px'}
px={3}
py={2}
>
{!!query && (
<Box whiteSpace={'pre-wrap'} mb={queryImages.length > 0 ? 2 : 0}>
{query}
</Box>
)}
<ImagePreviewToken images={queryImages} datasetId={datasetId} />
</Box>
);
});
/* Per response value */
export const WholeResponseContent = ({
activeModule,
......@@ -50,6 +82,7 @@ export const WholeResponseContent = ({
onOpenRequestIdDetail?: (requestId: string) => void;
}) => {
const { t } = useSafeTranslation();
const queryPreviewDatasetId = activeModule?.quoteList?.[0]?.datasetId;
// Auto scroll to top
const ContentRef = useRef<HTMLDivElement>(null);
......@@ -285,7 +318,20 @@ export const WholeResponseContent = ({
)}
<Row label={t('chat:step_query')} value={activeModule?.stepQuery} />
<Row label={t('common:core.chat.response.module query')} value={activeModule?.query} />
{activeModule?.queryImages && activeModule.queryImages.length > 0 ? (
<Row
label={t('common:core.chat.response.module query')}
rawDom={
<QueryWithImages
query={activeModule?.query}
queryImages={activeModule.queryImages}
datasetId={queryPreviewDatasetId}
/>
}
/>
) : (
<Row label={t('common:core.chat.response.module query')} value={activeModule?.query} />
)}
<Row
label={t('common:core.chat.response.context total length')}
value={activeModule?.contextTotalLen}
......@@ -333,10 +379,7 @@ export const WholeResponseContent = ({
label={t('common:core.dataset.search.search mode')}
rawDom={
<Flex border={'base'} borderRadius={'md'} p={2}>
<Box>
{/* @ts-ignore */}
{t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
</Box>
<Box>{t(DatasetSearchModeMap[activeModule.searchMode]?.title as any)}</Box>
{activeModule.embeddingWeight && (
<>{`(${t('chat:response_hybrid_weight', {
emb: activeModule.embeddingWeight,
......@@ -838,7 +881,7 @@ export const ResponseBox = React.memo(function ResponseBox({
/* Format response data to slider data */
function pretreatmentResponse(res: ChatHistoryItemResType[]): sideTabItemType[] {
return res.map((item) => {
let children: sideTabItemType[] = [];
const children: sideTabItemType[] = [];
if (item?.toolDetail) children.push(...pretreatmentResponse(item?.toolDetail));
if (item?.pluginDetail) children.push(...pretreatmentResponse(item?.pluginDetail));
......
projects/app/src/components/core/dataset/ImagePreviewToken.tsx 0 → 100644
import React, { useEffect, useState } from 'react';
import { Box, Flex, Portal, type BoxProps, type FlexProps } from '@chakra-ui/react';
import { isDatasetFileObjectKey } from '@fastgpt/global/core/dataset/utils';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useSafeTranslation } from '@fastgpt/web/hooks/useSafeTranslation';
import { postGetSearchTestImagePreviewUrls } from '@/web/core/dataset/api/file';
export type ImagePreviewTokenItemType = {
key?: string;
url?: string;
previewUrl?: string;
name?: string;
};
const getDirectPreviewUrl = (image: ImagePreviewTokenItemType) => {
const url = image.previewUrl || image.url || '';
return url && !isDatasetFileObjectKey(url) ? url : '';
};
const ImagePreview = React.memo(function ImagePreview({
image,
datasetId
}: {
image: ImagePreviewTokenItemType;
datasetId?: string;
}) {
const { t } = useSafeTranslation();
const [previewUrl, setPreviewUrl] = useState(() => getDirectPreviewUrl(image));
const [loadFailed, setLoadFailed] = useState(false);
const [hasRefreshed, setHasRefreshed] = useState(false);
useEffect(() => {
setPreviewUrl(getDirectPreviewUrl(image));
setLoadFailed(false);
setHasRefreshed(false);
}, [image]);
useEffect(() => {
if (!image.key || !datasetId || hasRefreshed || (previewUrl && !loadFailed)) return;
let canceled = false;
setHasRefreshed(true);
postGetSearchTestImagePreviewUrls({
datasetId,
keys: [image.key]
})
.then((res) => {
const nextPreviewUrl = res.find((item) => item.key === image.key)?.previewUrl;
if (!canceled && nextPreviewUrl) {
setPreviewUrl(nextPreviewUrl);
setLoadFailed(false);
}
})
.catch(() => {});
return () => {
canceled = true;
};
}, [datasetId, hasRefreshed, image.key, loadFailed, previewUrl]);
if (previewUrl && !loadFailed) {
return (
<Box
as={'img'}
src={previewUrl}
alt={image.name || ''}
w={'80px'}
h={'80px'}
objectFit={'cover'}
borderRadius={'sm'}
onError={() => setLoadFailed(true)}
/>
);
}
return (
<Flex
w={'80px'}
h={'80px'}
flexDir={'column'}
alignItems={'center'}
justifyContent={'center'}
gap={1}
bg={'myGray.50'}
border={'1px dashed'}
borderColor={'myGray.300'}
borderRadius={'sm'}
color={'myGray.500'}
fontSize={'xs'}
lineHeight={'16px'}
>
<MyIcon name={'image'} w={'20px'} h={'20px'} color={'myGray.400'} />
<Box>{t('common:core.dataset.test.image_expired')}</Box>
</Flex>
);
});
const defaultTokenStyles: BoxProps = {
as: 'span',
display: 'inline-flex',
px: 2,
py: 1,
border: '1px solid',
borderColor: 'myGray.200',
borderRadius: 'md',
bg: 'white',
color: 'myGray.700',
cursor: 'default',
lineHeight: '16px',
verticalAlign: 'baseline'
};
const ImagePreviewToken = React.memo(function ImagePreviewToken({
images,
datasetId,
containerProps,
tokenProps
}: {
images: ImagePreviewTokenItemType[];
datasetId?: string;
containerProps?: FlexProps;
tokenProps?: BoxProps;
}) {
const { t } = useSafeTranslation();
const [hoveredImage, setHoveredImage] = useState<
| {
image: ImagePreviewTokenItemType;
top: number;
left: number;
}
| undefined
>();
if (images.length === 0) return null;
return (
<>
<Flex flexWrap={'wrap'} gap={2} {...containerProps}>
{images.map((image, index) => (
<Box
key={`${image.key || image.url || image.previewUrl || index}`}
{...defaultTokenStyles}
{...tokenProps}
onMouseEnter={(e) => {
const rect = e.currentTarget.getBoundingClientRect();
setHoveredImage({
image,
top: rect.bottom + 8,
left: rect.left
});
}}
onMouseLeave={() => setHoveredImage(undefined)}
>
{t('common:core.dataset.test.image_token')}
</Box>
))}
</Flex>
{!!hoveredImage && (
<Portal>
<Flex
position={'fixed'}
zIndex={'tooltip'}
top={`${hoveredImage.top}px`}
left={`${hoveredImage.left}px`}
p={3}
bg={'white'}
borderWidth={'1px'}
borderColor={'borderColor.base'}
borderRadius={'md'}
boxShadow={'2'}
pointerEvents={'none'}
>
<ImagePreview image={hoveredImage.image} datasetId={datasetId} />
</Flex>
</Portal>
)}
</>
);
});
export default ImagePreviewToken;
projects/app/src/components/core/dataset/QuoteItem.tsx
......@@ -12,9 +12,15 @@ import { SearchScoreTypeEnum, SearchScoreTypeMap } from '@fastgpt/global/core/da
import type { ReadCollectionSourceBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api';
import Markdown from '@/components/Markdown';
const InputDataModal = dynamic(() => import('@/pageComponents/dataset/detail/InputDataModal'));
const InputDataModal = dynamic(
() => import('@/pageComponents/dataset/detail/components/InputDataModal')
);
export type ScoreItemType = SearchDataResponseItemType['score'][0];
export type ScoreItemType = {
type: SearchScoreTypeEnum;
value: number;
index: number;
};
export const scoreTheme: Record<
string,
{
......@@ -52,32 +58,19 @@ export const formatScore = (score: ScoreItemType[]) => {
};
}
// rrf -> rerank -> embedding -> fullText 优先级
let rrfScore: ScoreItemType | undefined = undefined;
let reRankScore: ScoreItemType | undefined = undefined;
let embeddingScore: ScoreItemType | undefined = undefined;
let fullTextScore: ScoreItemType | undefined = undefined;
score.forEach((item) => {
if (item.type === SearchScoreTypeEnum.rrf) {
rrfScore = item;
} else if (item.type === SearchScoreTypeEnum.reRank) {
reRankScore = item;
} else if (item.type === SearchScoreTypeEnum.embedding) {
embeddingScore = item;
} else if (item.type === SearchScoreTypeEnum.fullText) {
fullTextScore = item;
}
});
const scoreList = [
SearchScoreTypeEnum.rrf,
SearchScoreTypeEnum.reRank,
SearchScoreTypeEnum.embedding,
SearchScoreTypeEnum.fullText
]
.map((type) => score.find((item) => item.type === type))
.filter((item): item is ScoreItemType => !!item);
const primaryScore = (rrfScore ||
reRankScore ||
embeddingScore ||
fullTextScore) as unknown as ScoreItemType;
const secondaryScore = [rrfScore, reRankScore, embeddingScore, fullTextScore].filter(
// @ts-ignore
(item) => item && primaryScore && item.type !== primaryScore.type
) as unknown as ScoreItemType[];
const primaryScore = scoreList[0];
const secondaryScore = primaryScore
? scoreList.slice(1).filter((item) => item.type !== primaryScore.type)
: [];
return {
primaryScore,
......@@ -142,41 +135,49 @@ const QuoteItem = ({
</Flex>
</MyTooltip>
)}
{score.secondaryScore.map((item, i) => (
<MyTooltip key={item.type} label={t(SearchScoreTypeMap[item.type]?.desc as any)}>
<Box fontSize={'xs'}>
<Flex alignItems={'flex-start'} lineHeight={1.2} mb={1}>
<Box
px={'5px'}
borderWidth={'1px'}
borderRadius={'sm'}
mr={'2px'}
{...(scoreTheme[i] && scoreTheme[i])}
>
<Box transform={'scale(0.9)'}>#{item.index + 1}</Box>
</Box>
<Box transform={'scale(0.9)'}>
{t(SearchScoreTypeMap[item.type]?.label as any)}: {item.value.toFixed(4)}
</Box>
</Flex>
<Box h={'4px'}>
{SearchScoreTypeMap[item.type]?.showScore && (
<Progress
value={item.value * 100}
h={'4px'}
w={'100%'}
size="sm"
borderRadius={'20px'}
{...(scoreTheme[i] && {
colorScheme: scoreTheme[i].colorScheme
{score.secondaryScore.map((item, i) => {
const theme = scoreTheme[i];
return (
<MyTooltip key={item.type} label={t(SearchScoreTypeMap[item.type]?.desc as any)}>
<Box fontSize={'xs'}>
<Flex alignItems={'flex-start'} lineHeight={1.2} mb={1}>
<Box
px={'5px'}
borderWidth={'1px'}
borderRadius={'sm'}
mr={'2px'}
{...(theme && {
color: theme.color,
bg: theme.bg,
borderColor: theme.borderColor
})}
bg="#E8EBF0"
/>
)}
>
<Box transform={'scale(0.9)'}>#{item.index + 1}</Box>
</Box>
<Box transform={'scale(0.9)'}>
{t(SearchScoreTypeMap[item.type]?.label as any)}: {item.value.toFixed(4)}
</Box>
</Flex>
<Box h={'4px'}>
{SearchScoreTypeMap[item.type]?.showScore && (
<Progress
value={item.value * 100}
h={'4px'}
w={'100%'}
size="sm"
borderRadius={'20px'}
{...(theme && {
colorScheme: theme.colorScheme
})}
bg="#E8EBF0"
/>
)}
</Box>
</Box>
</Box>
</MyTooltip>
))}
</MyTooltip>
);
})}
</Flex>
<Box flex={'1 0 0'}>
......
projects/app/src/components/core/dataset/SearchParamsTip.tsx
......@@ -5,7 +5,7 @@ import {
DatasetSearchModeMap
} from '@fastgpt/global/core/dataset/constants';
import { useTranslation } from 'next-i18next';
import React, { useEffect, useMemo } from 'react';
import React, { useMemo } from 'react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { getWebLLMModel } from '@/web/common/system/utils';
......@@ -45,6 +45,15 @@ const SearchParamsTip = ({
borderWidth={'1px'}
borderColor={'primary.1'}
sx={{
'& thead > tr': {
borderBottom: 'none !important'
},
'& thead > tr > th': {
borderBottom: 'none !important'
},
'& tbody > tr:first-of-type > td, & tbody > tr:first-of-type > th': {
borderTop: 'none !important'
},
'&::-webkit-scrollbar': {
height: '6px',
borderRadius: '4px'
......
projects/app/src/pageComponents/account/model/AddModelBox.tsx
......@@ -838,12 +838,11 @@ export const ModelEditModal = ({
data.priceTiers = priceTiers as any;
}
for (const key in data) {
// @ts-ignore
const val = data[key];
const modelData = data as Record<string, unknown>;
for (const key of Object.keys(modelData)) {
const val = modelData[key];
if (val === null || val === undefined || Number.isNaN(val)) {
// @ts-ignore
data[key] = '';
modelData[key] = '';
}
}
......@@ -915,7 +914,7 @@ export const ModelEditModal = ({
headerPx={'32px'}
>
<ModalBody px={'32px'} py={0}>
<Section title={t('account:model.basic_config_section')}>
<Section key={key} title={t('account:model.basic_config_section')}>
<Flex direction={['column', 'row']} gap={[6, 8]} alignItems={['stretch', 'flex-start']}>
<Grid flex={'1 0 0'} templateColumns={['1fr', 'repeat(2, minmax(0, 1fr))']} gap={4}>
<Field label={t('account:model.model_id')} tip={t('account:model.model_id_tip')}>
......@@ -1101,6 +1100,19 @@ export const ModelEditModal = ({
</Section>
)}
{isEmbeddingModel && (
<Section title={t('account:model.feature_config_section')}>
<Grid templateColumns={['1fr', 'repeat(2, minmax(0, 1fr))']} gap={4}>
<SwitchField
label={t('account:model.vision')}
tip={t('account:model.embedding_vision_tip')}
field={'vision'}
register={register}
/>
</Grid>
</Section>
)}
{priceUnit && feConfigs?.isPlus && (
<Section title={t('account:model.price_config_section')}>
{isLLMModel ? (
......
projects/app/src/pageComponents/account/model/ModelConfigTable.tsx
......@@ -11,13 +11,12 @@ import {
Tr,
Switch,
ModalBody,
Input,
ModalFooter,
Button,
useDisclosure
} from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import React, { useCallback, useMemo, useRef, useState } from 'react';
import React, { useCallback, useMemo, useState } from 'react';
import MySelect from '@fastgpt/web/components/common/MySelect';
import { modelTypeList, ModelTypeEnum } from '@fastgpt/global/core/ai/constants';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
......@@ -63,24 +62,30 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
const isRoot = userInfo?.username === 'root';
const [provider, setProvider] = useState<string | ''>('');
const providerList = useRef<{ label: React.ReactNode; value: string | '' }[]>([
{ label: t('common:All'), value: '' },
...getModelProviders(i18n.language).map((item) => ({
label: (
<HStack>
<Avatar src={item.avatar} w={'1rem'} />
<Box>{item.name}</Box>
</HStack>
),
value: item.id
}))
]);
const providerList = useMemo<{ label: React.ReactNode; value: string | '' }[]>(
() => [
{ label: t('common:All'), value: '' },
...getModelProviders(i18n.language).map((item) => ({
label: (
<HStack>
<Avatar src={item.avatar} w={'1rem'} />
<Box>{item.name}</Box>
</HStack>
),
value: item.id
}))
],
[getModelProviders, i18n.language, t]
);
const [modelType, setModelType] = useState<ModelTypeEnum | ''>('');
const selectModelTypeList = useRef<{ label: string; value: ModelTypeEnum | '' }[]>([
{ label: t('common:All'), value: '' },
...modelTypeList.map((item) => ({ label: t(item.label), value: item.value }))
]);
const selectModelTypeList = useMemo<{ label: string; value: ModelTypeEnum | '' }[]>(
() => [
{ label: t('common:All'), value: '' },
...modelTypeList.map((item) => ({ label: t(item.label), value: item.value }))
],
[t]
);
const [search, setSearch] = useState('');
const [showActive, setShowActive] = useState(false);
......@@ -233,10 +238,8 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
const filterProviderList = useMemo(() => {
const allProviderIds: string[] = systemModelList.map((model) => model.provider);
return providerList.current.filter(
(item) => allProviderIds.includes(item.value) || item.value === ''
);
}, [systemModelList]);
return providerList.filter((item) => allProviderIds.includes(item.value) || item.value === '');
}, [providerList, systemModelList]);
const { runAsync: onTestModel, loading: testingModel } = useRequest(getTestModel, {
manual: true,
......@@ -263,7 +266,7 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
const onCreateModel = (type: ModelTypeEnum) => {
const defaultModel = defaultModels[type];
setEditModelData({
const modelData = {
...defaultModel,
model: '',
name: '',
......@@ -278,9 +281,10 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
isDefault: false,
isDefaultDatasetTextModel: false,
isDefaultDatasetImageModel: false,
// @ts-ignore
type
});
} as SystemModelItemType;
setEditModelData(modelData);
};
const {
......@@ -337,7 +341,7 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
bg={'myGray.50'}
value={modelType}
onChange={setModelType}
list={selectModelTypeList.current}
list={selectModelTypeList}
/>
</HStack>
<Box flex={1} />
......@@ -381,7 +385,7 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
</Tr>
</Thead>
<Tbody>
{modelList.map((item, index) => (
{modelList.map((item) => (
<Tr key={item.model} _hover={{ bg: 'myGray.50' }}>
<Td fontSize={'sm'}>
<HStack>
......@@ -405,7 +409,9 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
)}
{item.vision && (
<MyTag type="borderFill" colorSchema="green" py={0.5}>
{t('account:model.vision_tag')}
{item.type === ModelTypeEnum.llm
? t('account:model.vision_tag')
: t('common:core.ai.model.multimodal')}
</MyTag>
)}
{item.toolChoice && (
......
projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx
......@@ -5,7 +5,7 @@ import { type Dispatch, type MutableRefObject, type SetStateAction, useState } f
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useCopyData } from '@fastgpt/web/hooks/useCopyData';
import InputDataModal from '@/pageComponents/dataset/detail/InputDataModal';
import InputDataModal from '@/pageComponents/dataset/detail/components/InputDataModal';
const CollectionQuoteItem = ({
quoteRefs,
......
projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx
......@@ -17,24 +17,32 @@ import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { FolderIcon } from '@fastgpt/global/common/file/image/constants';
import type { ApiDatasetServerType } from '@fastgpt/global/core/dataset/apiDataset/type';
const ApiDatasetForm = ({
type,
datasetId,
form
form,
controlWidth
}: {
type: `${DatasetTypeEnum}`;
datasetId?: string;
form: UseFormReturn<
{
apiDatasetServer?: ApiDatasetServerType;
},
any
>;
controlWidth?: string | string[];
form: UseFormReturn<any>;
}) => {
const { t } = useTranslation();
const { register, setValue, watch } = form;
const inputLayoutProps = controlWidth
? {
w: controlWidth,
flex: '0 0 auto' as const
}
: {};
const rowLayoutProps = controlWidth
? {
justifyContent: 'space-between' as const
}
: {};
const rowTopMargin = controlWidth ? 4 : 6;
const apiDatasetServer = watch('apiDatasetServer');
const yuqueServer = apiDatasetServer?.yuqueServer;
......@@ -114,25 +122,39 @@ const ApiDatasetForm = ({
closeBaseurlSelectModal();
};
const renderBaseUrlSelector = () => (
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'}>
Base URL
</FormLabel>
<MyBox py={1} fontSize={'sm'} flex={'1 0 0'} overflow="auto" isLoading={isFetching}>
{pathNames}
</MyBox>
const renderBaseUrlSelector = () => {
const selectorContent = (
<>
<MyBox py={1} fontSize={'sm'} flex={'1 0 0'} overflow="auto" isLoading={isFetching}>
{pathNames}
</MyBox>
<Button
ml={2}
variant={'whiteBase'}
onClick={openBaseurlSeletModal}
isDisabled={!canSelectBaseUrl}
>
{t('dataset:selectDirectory')}
</Button>
</Flex>
);
<Button
ml={2}
variant={'whiteBase'}
onClick={openBaseurlSeletModal}
isDisabled={!canSelectBaseUrl}
>
{t('dataset:selectDirectory')}
</Button>
</>
);
return (
<Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'}>
Base URL
</FormLabel>
{controlWidth ? (
<Flex alignItems={'center'} w={controlWidth}>
{selectorContent}
</Flex>
) : (
selectorContent
)}
</Flex>
);
};
// Render the directory selection modal
const renderDirectoryModal = () =>
......@@ -156,22 +178,24 @@ const ApiDatasetForm = ({
<>
{type === DatasetTypeEnum.apiDataset && (
<>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
<Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
{t('dataset:api_url')}
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={t('dataset:api_url')}
maxLength={200}
{...register('apiDatasetServer.apiServer.baseUrl', { required: true })}
/>
</Flex>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
<Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
Authorization
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
maxLength={2000}
......@@ -184,51 +208,36 @@ const ApiDatasetForm = ({
)}
{type === DatasetTypeEnum.feishu && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={rowTopMargin} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
App ID
</Flex>
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={'App ID'}
maxLength={200}
{...register('apiDatasetServer.feishuServer.appId', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={rowTopMargin} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
App Secret
</Flex>
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={'App Secret'}
maxLength={200}
{...register('apiDatasetServer.feishuServer.appSecret', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={rowTopMargin} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
Folder Token
</Flex>
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={'Folder Token'}
maxLength={200}
......@@ -241,22 +250,24 @@ const ApiDatasetForm = ({
)}
{type === DatasetTypeEnum.yuque && (
<>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
<Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
User ID
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={'User ID'}
maxLength={200}
{...register('apiDatasetServer.yuqueServer.userId', { required: true })}
/>
</Flex>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
<Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
Token
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={'Token'}
maxLength={200}
......@@ -269,33 +280,36 @@ const ApiDatasetForm = ({
)}
{type === DatasetTypeEnum.dingtalk && (
<>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
<Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
App Key
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={'App Key'}
maxLength={200}
{...register('apiDatasetServer.dingtalkServer.appKey', { required: true })}
/>
</Flex>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
<Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
App Secret
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={'App Secret'}
maxLength={200}
{...register('apiDatasetServer.dingtalkServer.appSecret', { required: true })}
/>
</Flex>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
<Flex mt={rowTopMargin} alignItems={'center'} {...rowLayoutProps}>
<FormLabel required flex={['', '0 0 110px']} fontSize={'sm'}>
User ID
</FormLabel>
<Input
{...inputLayoutProps}
bg={'myWhite.600'}
placeholder={'User ID'}
maxLength={200}
......
projects/app/src/pageComponents/dataset/detail/DataCard.tsx
import React, { useState, useMemo } from 'react';
import { Box, Card, IconButton, Flex, Button, useTheme, Image } from '@chakra-ui/react';
import { Box, Card, IconButton, Flex, Button } from '@chakra-ui/react';
import { getDatasetCollectionById } from '@/web/core/dataset/api/collection';
import { getDatasetDataList, delOneDatasetDataById } from '@/web/core/dataset/api/data';
import { useToast } from '@fastgpt/web/hooks/useToast';
......@@ -8,7 +8,7 @@ import { useTranslation } from 'next-i18next';
import { useRouter } from 'next/router';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyInput from '@/components/MyInput';
import InputDataModal from './InputDataModal';
import InputDataModal from './components/InputDataModal';
import RawSourceBox from '@/components/core/dataset/RawSourceBox';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
......@@ -175,7 +175,8 @@ const DataCard = () => {
isDisabled={!collection}
isLoading={isExportChunksLoading}
onClick={() => {
onExportAllChunks(collection?._id!);
if (!collection?._id) return;
onExportAllChunks(collection._id);
}}
>
{t('dataset:collection.export_all_chunks')}
......
projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx
......@@ -135,6 +135,45 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
const imageIndex = watch('imageIndex');
const indexPrefixTitle = watch('indexPrefixTitle');
const paragraphChunkAIMode = watch('paragraphChunkAIMode');
const imageIndexConfigState = useMemo(() => {
if (!feConfigs?.isPlus) {
return {
disabled: true,
tooltip: t('common:commercial_function_tip'),
tip: t('dataset:image_auto_parse_tip_commercial')
};
}
if (datasetDetail.vectorModel?.vision && datasetDetail.vlmModel) {
return {
disabled: false,
tooltip: '',
tip: t('dataset:image_auto_parse_tip_multimodal_with_vlm')
};
}
if (datasetDetail.vectorModel?.vision) {
return {
disabled: false,
tooltip: '',
tip: t('dataset:image_auto_parse_tip_multimodal_without_vlm')
};
}
if (datasetDetail.vlmModel) {
return {
disabled: false,
tooltip: '',
tip: t('dataset:image_auto_parse_tip_vlm_only')
};
}
return {
disabled: true,
tooltip: t('dataset:image_auto_parse_tip_no_vlm_or_multimodal'),
tip: t('dataset:image_auto_parse_tip_no_vlm_or_multimodal')
};
}, [datasetDetail.vectorModel?.vision, datasetDetail.vlmModel, feConfigs?.isPlus, t]);
const trainingModeList = useMemo(() => {
const list = {
......@@ -224,6 +263,12 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
}
}, [trainingType, setValue]);
useEffect(() => {
if (imageIndexConfigState.disabled && imageIndex) {
setValue('imageIndex', false);
}
}, [imageIndex, imageIndexConfigState.disabled, setValue]);
return (
<>
<Box>
......@@ -310,24 +355,16 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
<QuestionTip label={t('dataset:auto_indexes_tips')} />
</HStack>
<HStack flex={'1'} spacing={1}>
<MyTooltip
label={
!feConfigs?.isPlus
? t('common:commercial_function_tip')
: !datasetDetail?.vlmModel
? t('common:error_vlm_not_config')
: ''
}
>
<MyTooltip label={imageIndexConfigState.tooltip}>
<Checkbox
isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
isDisabled={imageIndexConfigState.disabled}
isChecked={imageIndex}
{...register('imageIndex')}
>
<FormLabel>{t('dataset:image_auto_parse')}</FormLabel>
</Checkbox>
</MyTooltip>
<QuestionTip label={t('dataset:image_auto_parse_tips')} />
<QuestionTip label={imageIndexConfigState.tip} />
</HStack>
</>
)}
......
projects/app/src/pageComponents/dataset/detail/Info/components/EditApiServiceModal.tsx
......@@ -41,7 +41,7 @@ const EditAPIDatasetInfoModal = ({
const { runAsync: onSave, loading } = useRequest(
(data: EditAPIDatasetInfoFormType) => onEdit(data),
{
onSuccess: (res) => {
onSuccess: () => {
toast({
title: t('common:update_success'),
status: 'success'
......@@ -72,7 +72,6 @@ const EditAPIDatasetInfoModal = ({
</Flex>
</Flex>
)}
{/* @ts-ignore */}
<ApiDatasetForm datasetId={datasetDetail._id} type={type} form={form} />
</ModalBody>
<ModalFooter>
......
projects/app/src/pageComponents/dataset/detail/Info/index.tsx
......@@ -176,7 +176,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
</Box>
</MyTooltip>
</Flex>
<Box pt={2}>
<Box pt={2} minW={0} maxW={'100%'} overflow={'hidden'}>
<AIModelSelector
w={'100%'}
value={vectorModel.model}
......@@ -210,7 +210,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('common:core.ai.model.Dataset Agent Model')}
</FormLabel>
<Box pt={2}>
<Box pt={2} minW={0} maxW={'100%'} overflow={'hidden'}>
<AIModelSelector
w={'100%'}
value={agentModel.model}
......@@ -233,7 +233,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:vllm_model')}
</FormLabel>
<Box pt={2}>
<Box pt={2} minW={0} maxW={'100%'} overflow={'hidden'}>
<AIModelSelector
w={'100%'}
value={vlmModel?.model}
......
projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx deleted 100644 → 0
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Box, Flex, Button, Textarea, ModalFooter, HStack, VStack, Image } from '@chakra-ui/react';
import type { UseFormRegister } from 'react-hook-form';
import { useFieldArray, useForm } from 'react-hook-form';
import { getDatasetCollectionById } from '@/web/core/dataset/api/collection';
import {
postInsertData2Dataset,
putDatasetDataById,
getDatasetDataItemById
} from '@/web/core/dataset/api/data';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useTranslation } from 'next-i18next';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import type { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
import DeleteIcon from '@fastgpt/web/components/common/Icon/delete';
import { defaultCollectionDetail } from '@/web/core/dataset/constants';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import styles from './styles.module.scss';
import {
DatasetDataIndexTypeEnum,
getDatasetIndexMapData
} from '@fastgpt/global/core/dataset/data/constants';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyIconButton from '@fastgpt/web/components/common/Icon/button';
import MyImage from '@/components/MyImage/index';
export type InputDataType = {
q: string;
a: string;
imagePreivewUrl?: string;
indexes: (Omit<DatasetDataIndexItemType, 'dataId'> & {
dataId?: string; // pg data id
fold: boolean;
})[];
};
enum TabEnum {
chunk = 'chunk',
qa = 'qa',
image = 'image'
}
const InputDataModal = ({
collectionId,
dataId,
defaultValue,
onClose,
onSuccess
}: {
collectionId: string;
dataId?: string;
defaultValue?: { q?: string; a?: string; imagePreivewUrl?: string };
onClose: () => void;
onSuccess: (data: InputDataType & { dataId: string }) => void;
}) => {
const { t } = useTranslation();
const { embeddingModelList, defaultModels } = useSystemStore();
const [currentTab, setCurrentTab] = useState<TabEnum>();
const { register, handleSubmit, reset, control, watch } = useForm<InputDataType>();
const {
fields: indexes,
prepend: prependIndexes,
remove: removeIndexes,
update: updateIndexes
} = useFieldArray({
control,
name: 'indexes'
});
const imagePreivewUrl = watch('imagePreivewUrl');
const { data: collection = defaultCollectionDetail, loading: initLoading } = useRequest(
async () => {
const [collection, dataItem] = await Promise.all([
getDatasetCollectionById(collectionId),
...(dataId ? [getDatasetDataItemById(dataId)] : [])
]);
if (dataItem) {
setCurrentTab(dataItem?.a ? TabEnum.qa : TabEnum.chunk);
reset({
q: dataItem.q || '',
a: dataItem.a || '',
imagePreivewUrl: dataItem.imagePreivewUrl,
indexes: dataItem.indexes.map((item) => ({
...item,
fold: true
}))
});
} else if (defaultValue) {
setCurrentTab(defaultValue?.a ? TabEnum.qa : TabEnum.chunk);
reset({
q: defaultValue.q || '',
a: defaultValue.a || '',
imagePreivewUrl: defaultValue.imagePreivewUrl
});
} else {
setCurrentTab(TabEnum.chunk);
}
// Forcus reset to image tab
if (collection.type === DatasetCollectionTypeEnum.images) {
setCurrentTab(TabEnum.image);
}
return collection;
},
{
manual: false,
refreshDeps: [collectionId, dataId, defaultValue]
}
);
// Import new data
const { runAsync: sureImportData, loading: isImporting } = useRequest(
async (e: InputDataType) => {
const data = { ...e };
const postData: any = {
collectionId: collection._id,
q: e.q,
a: currentTab === TabEnum.qa ? e.a : '',
// Contains no default index
indexes: e.indexes?.filter((item) => !!item.text?.trim()) || []
};
const dataId = await postInsertData2Dataset(postData);
return {
...data,
dataId
};
},
{
refreshDeps: [currentTab],
successToast: t('common:dataset.data.Input Success Tip'),
onSuccess(e) {
reset({
...e,
q: '',
a: '',
indexes: []
});
onSuccess(e);
},
errorToast: t('dataset:common.error.unKnow')
}
);
// Update data
const { runAsync: onUpdateData, loading: isUpdating } = useRequest(
async (e: InputDataType) => {
if (!dataId) return Promise.reject(t('common:error.unKnow'));
const updateData: any = {
dataId,
q: e.q,
a: currentTab === TabEnum.qa ? e.a : '',
indexes: e.indexes.filter((item) => !!item.text?.trim())
};
await putDatasetDataById(updateData);
return {
dataId,
...e
};
},
{
refreshDeps: [currentTab],
successToast: t('common:dataset.data.Update Success Tip'),
onSuccess(data) {
onSuccess(data);
onClose();
}
}
);
const icon = useMemo(
() => getCollectionIcon({ type: collection.type, name: collection.sourceName }),
[collection]
);
const maxToken = useMemo(() => {
const vectorModel =
embeddingModelList.find((item) => item.model === collection.dataset.vectorModel) ||
defaultModels.embedding;
return vectorModel?.maxToken || 2000;
}, [collection.dataset.vectorModel, defaultModels.embedding, embeddingModelList]);
return (
<MyModal
isOpen={true}
isCentered
w={['20rem', '64rem']}
onClose={() => onClose()}
closeOnOverlayClick={false}
maxW={'1440px'}
h={'46.25rem'}
title={
<Flex ml={-3}>
<MyIcon name={icon as any} w={['16px', '20px']} mr={2} />
<Box
className={'textEllipsis'}
wordBreak={'break-all'}
fontSize={'md'}
maxW={['200px', '50vw']}
fontWeight={'500'}
color={'myGray.900'}
whiteSpace={'nowrap'}
overflow={'hidden'}
textOverflow={'ellipsis'}
>
{collection.sourceName || t('common:unknow_source')}
</Box>
</Flex>
}
>
<MyBox
display={'flex'}
flexDir={'column'}
isLoading={initLoading}
h={'100%'}
py={[6, '1.5rem']}
>
{/* Tab */}
<Box px={[5, '3.25rem']}>
{(currentTab === TabEnum.chunk || currentTab === TabEnum.qa) && (
<FillRowTabs
list={[
{ label: t('common:dataset_data_input_chunk'), value: TabEnum.chunk },
{ label: t('common:dataset_data_input_qa'), value: TabEnum.qa }
]}
py={1}
value={currentTab}
onChange={(e) => {
setCurrentTab(e);
}}
/>
)}
</Box>
<Flex flex={'1 0 0'} h={['auto', '0']} gap={6} flexDir={['column', 'row']} px={[5, '0']}>
{/* Data */}
<Flex
pt={4}
pl={[0, '3.25rem']}
flexDir={'column'}
h={'100%'}
gap={3}
flex={'1 0 0'}
w={['100%', 0]}
overflow={['unset', 'auto']}
>
<Flex flexDir={'column'} flex={'1 0 0'} h={0}>
{currentTab === TabEnum.image && (
<>
<FormLabel required mb={1} h={'30px'}>
{t('file:image')}
</FormLabel>
<Box flex={'1 0 0'} h={0} w="100%">
<Box height="100%" position="relative" border="base" borderRadius={'md'} p={1}>
<MyImage
src={imagePreivewUrl}
h="100%"
w="100%"
objectFit="contain"
alt={t('file:Image_Preview')}
/>
</Box>
</Box>
</>
)}
{(currentTab === TabEnum.chunk || currentTab === TabEnum.qa) && (
<>
<FormLabel required mb={1} h={'30px'}>
{currentTab === TabEnum.chunk
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
<Textarea
resize={'none'}
className={styles.scrollbar}
flex={'1 0 0'}
tabIndex={1}
_focus={{
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
bg={'myGray.25'}
borderRadius={'md'}
borderColor={'myGray.200'}
{...register(`q`, {
required: true
})}
/>
</>
)}
</Flex>
{currentTab === TabEnum.qa && (
<Flex flexDir={'column'} flex={'1 0 0'}>
<FormLabel required mb={1}>
{t('common:dataset_data_input_a')}
</FormLabel>
<Textarea
resize={'none'}
className={styles.scrollbar}
flex={'1 0 0'}
tabIndex={1}
bg={'myGray.25'}
borderRadius={'md'}
border={'1.5px solid '}
borderColor={'myGray.200'}
{...register('a', { required: true })}
/>
</Flex>
)}
{currentTab === TabEnum.image && (
<Flex flexDir={'column'} flex={'1 0 0'}>
<FormLabel required mb={1}>
{t('file:image_description')}
</FormLabel>
<Textarea
resize={'none'}
placeholder={t('file:image_description_tip')}
className={styles.scrollbar}
flex={'1 0 0'}
tabIndex={1}
bg={'myGray.25'}
borderRadius={'md'}
border={'1.5px solid '}
borderColor={'myGray.200'}
{...register('q', {
required: true
})}
/>
</Flex>
)}
</Flex>
{/* Index */}
<Box
pt={4}
pr={[0, '3.25rem']}
flex={'1 0 0'}
w={['100%', 0]}
overflow={['unset', 'auto']}
>
<Flex alignItems={'flex-start'} justifyContent={'space-between'} h={'30px'}>
<FormLabel>
{t('common:dataset.data.edit.Index', {
amount: indexes.length
})}
</FormLabel>
<Button
variant={'whiteBase'}
size={'sm'}
p={0}
transform={'translateY(-6px)'}
onClick={() =>
prependIndexes({
type: DatasetDataIndexTypeEnum.custom,
text: '',
fold: false
})
}
>
<Flex px={'0.62rem'} py={2}>
<MyIcon name={'common/addLight'} w={'1rem'} mr={'0.38rem'} />
{t('common:add_new')}
</Flex>
</Button>
</Flex>
<VStack>
{indexes?.map((index, i) => {
const data = getDatasetIndexMapData(index.type);
return (
<Box
key={index.dataId || i}
p={4}
borderRadius={'md'}
border={'base'}
bg={'myGray.25'}
w={'100%'}
_hover={{
'& .delete': {
display: 'block'
}
}}
>
{/* Header */}
<Flex mb={2} alignItems={'center'}>
<FormLabel flex={'1 0 0'}>{t(data.label)}</FormLabel>
{/* Delete */}
{index.type !== 'default' && (
<HStack className={'delete'} borderRight={'base'} pr={3} mr={2}>
<DeleteIcon
onClick={() => {
removeIndexes(i);
}}
/>
</HStack>
)}
{indexes.length > 1 && (
<MyIconButton
icon={index.fold ? 'core/chat/chevronDown' : 'core/chat/chevronUp'}
onClick={() => {
updateIndexes(i, { ...index, fold: !index.fold });
}}
/>
)}
</Flex>
{/* Content */}
<DataIndexTextArea
disabled={index.type === 'default'}
index={i}
value={index.text}
isFolder={index.fold && indexes.length > 1}
maxToken={maxToken}
register={register}
onFocus={() => {
updateIndexes(i, { ...index, fold: false });
}}
/>
</Box>
);
})}
</VStack>
</Box>
</Flex>
<ModalFooter px={[5, '3.25rem']} py={0} pt={4}>
<MyTooltip
label={collection.permission.hasWritePer ? '' : t('common:dataset.data.Can not edit')}
>
<Button
isDisabled={!collection.permission.hasWritePer}
isLoading={isImporting || isUpdating}
// @ts-ignore
onClick={handleSubmit(dataId ? onUpdateData : sureImportData)}
>
{dataId ? t('common:confirm_update') : t('common:comfirm_import')}
</Button>
</MyTooltip>
</ModalFooter>
</MyBox>
</MyModal>
);
};
export default React.memo(InputDataModal);
const textareaMinH = '40px';
const DataIndexTextArea = ({
value,
index,
maxToken,
register,
disabled,
isFolder,
onFocus
}: {
value: string;
index: number;
maxToken: number;
register: UseFormRegister<InputDataType>;
disabled?: boolean;
isFolder: boolean;
onFocus: () => void;
}) => {
const { t } = useTranslation();
const TextareaDom = useRef<HTMLTextAreaElement | null>(null);
const {
ref: TextareaRef,
required,
name,
onChange: onTextChange,
onBlur
} = register(`indexes.${index}.text`, { required: true });
useEffect(() => {
if (TextareaDom.current) {
TextareaDom.current.style.height = textareaMinH;
TextareaDom.current.style.height = `${TextareaDom.current.scrollHeight + 5}px`;
}
}, []);
const autoHeight = useCallback((e: React.ChangeEvent<HTMLTextAreaElement>) => {
if (e.target) {
e.target.style.height = textareaMinH;
e.target.style.height = `${e.target.scrollHeight + 5}px`;
}
}, []);
const onclickMark = () => {
TextareaDom?.current?.focus();
onFocus();
};
return (
<Box
pos={'relative'}
{...(isFolder
? {
maxH: '50px',
overflow: 'hidden'
}
: {
maxH: 'auto'
})}
>
{disabled ? (
<Box fontSize={'sm'} color={'myGray.500'} whiteSpace={'pre-wrap'}>
{value}
</Box>
) : (
<Textarea
maxLength={maxToken}
borderColor={'transparent'}
className={styles.scrollbar}
minH={textareaMinH}
px={0}
pt={0}
isRequired={required}
whiteSpace={'pre-wrap'}
resize={'none'}
_focus={{
px: 3,
py: 1,
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
placeholder={t('common:dataset.data.Index Placeholder')}
ref={(e) => {
if (e) TextareaDom.current = e;
TextareaRef(e);
}}
required
name={name}
onChange={(e) => {
autoHeight(e);
onTextChange(e);
}}
onFocus={autoHeight}
onBlur={onBlur}
/>
)}
{isFolder && (
<Box
pos={'absolute'}
bottom={0}
left={0}
right={0}
top={0}
bg={'linear-gradient(182deg, rgba(251, 251, 252, 0.00) 1.76%, #FBFBFC 84.07%)'}
{...(disabled
? {}
: {
cursor: 'pointer',
onClick: onclickMark
})}
/>
)}
</Box>
);
};
projects/app/src/pageComponents/dataset/detail/Test.tsx deleted 100644 → 0
import React, { useEffect, useMemo, useState } from 'react';
import { Box, Textarea, Button, Flex, useTheme, useDisclosure } from '@chakra-ui/react';
import {
useSearchTestStore,
type SearchTestStoreItemType
} from '@/web/core/dataset/store/searchTest';
import { postSearchText } from '@/web/core/dataset/api';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { formatTimeToChatTime } from '@fastgpt/global/common/string/time';
import { useToast } from '@fastgpt/web/hooks/useToast';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useTranslation } from 'next-i18next';
import { type SearchDatasetTestResponse } from '@fastgpt/global/openapi/core/dataset/api';
import {
DatasetSearchModeEnum,
DatasetSearchModeMap
} from '@fastgpt/global/core/dataset/constants';
import dynamic from 'next/dynamic';
import { useForm } from 'react-hook-form';
import MySelect from '@fastgpt/web/components/common/MySelect';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { fileDownload } from '@/web/common/file/utils';
import QuoteItem from '@/components/core/dataset/QuoteItem';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import SearchParamsTip from '@/components/core/dataset/SearchParamsTip';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import { getNanoid } from '@fastgpt/global/common/string/tools';
const DatasetParamsModal = dynamic(() => import('@/components/core/app/DatasetParamsModal'));
type FormType = {
inputText: string;
searchParams: {
searchMode: DatasetSearchModeEnum;
embeddingWeight?: number;
usingReRank?: boolean;
rerankModel?: string;
rerankWeight?: number;
similarity?: number;
limit?: number;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
};
};
const Test = ({ datasetId }: { datasetId: string }) => {
const { t } = useTranslation();
const { toast } = useToast();
const { defaultModels } = useSystemStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { pushDatasetTestItem } = useSearchTestStore();
const [inputType, setInputType] = useState<'text' | 'file'>('text');
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
const [isFocus, setIsFocus] = useState(false);
const { File, onOpen } = useSelectFile({
fileType: '.csv',
multiple: false
});
const [selectFile, setSelectFile] = useState<File>();
const { getValues, setValue, register, handleSubmit } = useForm<FormType>({
defaultValues: {
inputText: '',
searchParams: {
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: true,
rerankModel: defaultModels?.rerank?.model,
rerankWeight: 0.5,
limit: 5000,
similarity: 0,
datasetSearchUsingExtensionQuery: false,
datasetSearchExtensionModel: defaultModels.llm?.model,
datasetSearchExtensionBg: ''
}
}
});
const searchModeData = DatasetSearchModeMap[getValues(`searchParams.searchMode`)];
const searchParams = getValues('searchParams');
const {
isOpen: isOpenSelectMode,
onOpen: onOpenSelectMode,
onClose: onCloseSelectMode
} = useDisclosure();
const { runAsync: onTextTest, loading: textTestIsLoading } = useRequest(
({ inputText, searchParams }: FormType) =>
postSearchText({ datasetId, text: inputText.trim(), ...searchParams }),
{
onSuccess(res: SearchDatasetTestResponse) {
if (!res || res.list.length === 0) {
return toast({
status: 'warning',
title: t('common:dataset.test.noResult')
});
}
const testItem: SearchTestStoreItemType = {
id: getNanoid(),
datasetId,
text: getValues('inputText').trim(),
time: new Date(),
results: res.list,
duration: res.duration,
searchMode: res.searchMode,
usingReRank: res.usingReRank,
limit: res.limit,
similarity: res.similarity,
queryExtensionModel: res.queryExtensionModel
};
pushDatasetTestItem(testItem);
setDatasetTestItem(testItem);
}
}
);
const onSelectFile = async (files: File[]) => {
const file = files[0];
if (!file) return;
setSelectFile(file);
};
useEffect(() => {
setDatasetTestItem(undefined);
}, [datasetId]);
return (
<Box h={'100%'} display={['block', 'flex']}>
{/* left */}
<Box
h={['auto', '100%']}
display={['block', 'flex']}
flexDirection={'column'}
flex={1}
maxW={'500px'}
py={4}
>
<Box
border={'2px solid'}
p={3}
mx={4}
borderRadius={'md'}
{...(isFocus
? {
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)'
}
: {
borderColor: 'primary.300'
})}
>
{/* header */}
<Flex alignItems={'center'} justifyContent={'space-between'}>
<MySelect<'text' | 'file'>
size={'sm'}
list={[
{
label: (
<Flex alignItems={'center'}>
<MyIcon mr={2} name={'text'} w={'14px'} color={'primary.600'} />
<Box fontSize={'sm'} fontWeight={'bold'} flex={1}>
{t('common:core.dataset.test.Test Text')}
</Box>
</Flex>
),
value: 'text'
}
// {
// label: (
// <Flex alignItems={'center'}>
// <MyIcon mr={2} name={'file/csv'} w={'14px'} color={'primary.600'} />
// <Box fontSize={'sm'} fontWeight={'bold'} flex={1}>
// {t('common:core.dataset.test.Batch test')}
// </Box>
// </Flex>
// ),
// value: 'file'
// }
]}
value={inputType}
onChange={(e) => setInputType(e)}
/>
<Button
variant={'whitePrimary'}
leftIcon={<MyIcon name={searchModeData.icon as any} w={'14px'} />}
size={'sm'}
onClick={onOpenSelectMode}
>
{t(searchModeData.title as any)}
</Button>
</Flex>
<Box h={'180px'}>
{inputType === 'text' && (
<Textarea
h={'100%'}
resize={'none'}
variant={'unstyled'}
maxLength={datasetDetail.vectorModel?.maxToken}
placeholder={t('common:core.dataset.test.Test Text Placeholder')}
onFocus={() => setIsFocus(true)}
{...register('inputText', {
required: true,
onBlur: () => {
setIsFocus(false);
}
})}
/>
)}
{inputType === 'file' && (
<Box pt={5}>
<Flex
p={3}
borderRadius={'md'}
borderWidth={'1px'}
borderColor={'borderColor.base'}
borderStyle={'dashed'}
bg={'white'}
cursor={'pointer'}
justifyContent={'center'}
_hover={{
bg: 'primary.100',
borderColor: 'primary.500',
borderStyle: 'solid'
}}
onClick={onOpen}
>
<MyIcon mr={2} name={'file/csv'} w={'24px'} />
<Box>
{selectFile
? selectFile.name
: t('common:core.dataset.test.Batch test Placeholder')}
</Box>
</Flex>
<Box mt={3} fontSize={'sm'}>
{t('common:info.csv_message')}
<Box
as={'span'}
color={'primary.600'}
cursor={'pointer'}
onClick={() => {
fileDownload({
text: `"问题"\n"问题1"\n"问题2"\n"问题3"`,
type: 'text/csv',
filename: 'Test Template'
});
}}
>
{t('common:info.csv_download')}
</Box>
</Box>
</Box>
)}
</Box>
<Flex justifyContent={'flex-end'}>
<Button
size={'sm'}
isLoading={textTestIsLoading}
isDisabled={inputType === 'file' && !selectFile}
onClick={() => {
if (inputType === 'text') {
handleSubmit((data) => onTextTest(data))();
} else {
// handleSubmit((data) => onFileTest(data))();
}
}}
>
{t('common:core.dataset.test.Test')}
</Button>
</Flex>
</Box>
<Box mt={5} px={4} overflow={'overlay'} display={['none', 'block']}>
<TestHistories
datasetId={datasetId}
datasetTestItem={datasetTestItem}
setDatasetTestItem={setDatasetTestItem}
/>
</Box>
</Box>
{/* result show */}
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={'1 0 0'} bg={'white'}>
<TestResults datasetTestItem={datasetTestItem} />
</Box>
{isOpenSelectMode && (
<DatasetParamsModal
{...searchParams}
maxTokens={20000}
onClose={onCloseSelectMode}
onSuccess={(e) => {
setValue('searchParams', {
...searchParams,
...e
});
}}
/>
)}
<File onSelect={onSelectFile} />
</Box>
);
};
export default React.memo(Test);
const TestHistories = React.memo(function TestHistories({
datasetId,
datasetTestItem,
setDatasetTestItem
}: {
datasetId: string;
datasetTestItem?: SearchTestStoreItemType;
setDatasetTestItem: React.Dispatch<React.SetStateAction<SearchTestStoreItemType | undefined>>;
}) {
const { t } = useTranslation();
const { datasetTestList, delDatasetTestItemById } = useSearchTestStore();
const testHistories = useMemo(
() => datasetTestList.filter((item) => item.datasetId === datasetId),
[datasetId, datasetTestList]
);
return (
<>
<Flex alignItems={'center'} color={'myGray.900'}>
<MyIcon mr={2} name={'history'} w={'18px'} h={'18px'} color={'myGray.900'} />
<Box fontSize={'md'}>{t('common:core.dataset.test.test history')}</Box>
</Flex>
<Box mt={2}>
{testHistories.map((item) => (
<Flex
key={item.id}
py={2}
px={3}
alignItems={'center'}
borderColor={'borderColor.low'}
borderWidth={'1px'}
borderRadius={'md'}
_notLast={{
mb: 2
}}
_hover={{
borderColor: 'primary.300',
boxShadow: '1',
'& .delete': {
display: 'block'
},
'& .time': {
display: 'none'
}
}}
cursor={'pointer'}
fontSize={'sm'}
{...(item.id === datasetTestItem?.id && {
bg: 'primary.50'
})}
onClick={() => setDatasetTestItem(item)}
>
<Box flex={'0 0 auto'} mr={2}>
{DatasetSearchModeMap[item.searchMode] ? (
<Flex alignItems={'center'} fontWeight={'500'} color={'myGray.500'}>
<MyIcon
name={DatasetSearchModeMap[item.searchMode].icon as any}
w={'12px'}
mr={'1px'}
/>
{t(DatasetSearchModeMap[item.searchMode].title as any)}
</Flex>
) : (
'-'
)}
</Box>
<Box flex={1} mr={2} wordBreak={'break-all'} fontWeight={'400'}>
{item.text}
</Box>
<Box className="time" flex={'0 0 auto'} fontSize={'xs'} color={'myGray.500'}>
{t(formatTimeToChatTime(item.time) as any).replace('#', ':')}
</Box>
<MyTooltip label={t('common:core.dataset.test.delete test history')}>
<Box className="delete" display={'none'} w={'0.8rem'} h={'0.8rem'} ml={1}>
<MyIcon
name={'delete'}
w={'0.8rem'}
_hover={{ color: 'red.600' }}
onClick={(e) => {
e.stopPropagation();
delDatasetTestItemById(item.id);
datasetTestItem?.id === item.id && setDatasetTestItem(undefined);
}}
/>
</Box>
</MyTooltip>
</Flex>
))}
</Box>
</>
);
});
const TestResults = React.memo(function TestResults({
datasetTestItem
}: {
datasetTestItem?: SearchTestStoreItemType;
}) {
const { t } = useTranslation();
const theme = useTheme();
return (
<>
{!datasetTestItem?.results || datasetTestItem.results.length === 0 ? (
<EmptyTip text={t('common:core.dataset.test.test result placeholder')} mt={[10, '20vh']} />
) : (
<>
<Flex fontSize={'md'} color={'myGray.900'} alignItems={'center'}>
<MyIcon name={'common/paramsLight'} w={'18px'} mr={2} />
{t('common:core.dataset.test.Test params')}
</Flex>
<Box mt={3}>
<SearchParamsTip
searchMode={datasetTestItem.searchMode}
similarity={datasetTestItem.similarity}
limit={datasetTestItem.limit}
usingReRank={datasetTestItem.usingReRank}
usingExtensionQuery={!!datasetTestItem.queryExtensionModel}
queryExtensionModel={datasetTestItem.queryExtensionModel}
/>
</Box>
<Flex mt={5} mb={3} alignItems={'center'}>
<Flex fontSize={'md'} color={'myGray.900'} alignItems={'center'}>
<MyIcon name={'common/resultLight'} w={'18px'} mr={2} />
{t('common:core.dataset.test.Test Result')}
</Flex>
<QuestionTip ml={1} label={t('common:core.dataset.test.test result tip')} />
<Box ml={2}>({datasetTestItem.duration})</Box>
</Flex>
<Box mt={1} gap={4}>
{datasetTestItem?.results.map((item, index) => (
<Box key={item.id} p={3} borderRadius={'lg'} bg={'myGray.100'} _notLast={{ mb: 2 }}>
<QuoteItem quoteItem={item} canDownloadSource canEditData />
</Box>
))}
</Box>
</>
)}
</>
);
});
projects/app/src/pageComponents/dataset/detail/Test/components/SearchTestImagePreviewList.tsx 0 → 100644
import React from 'react';
import { Box, CircularProgress, Flex } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { searchTestImageThumbProps } from '../constants';
import type { SearchTestImageRef } from '../type';
type SearchTestImagePreviewListItem =
| (SearchTestImageRef & {
type: 'image';
})
| {
type: 'uploading';
key: string;
};
// Render persisted previews and upload placeholders in the same grid to avoid layout jumps.
const SearchTestImagePreviewList = ({
images,
uploadingCount,
onRemove
}: {
images: SearchTestImageRef[];
uploadingCount: number;
onRemove: (key: string) => void;
}) => {
if (images.length === 0 && uploadingCount === 0) return null;
return (
<Flex mb={3} gap={2} flexWrap={'wrap'}>
{images.map((image) => (
<SearchTestImagePreviewItem
key={image.key}
item={{ type: 'image', ...image }}
onRemove={onRemove}
/>
))}
{Array.from({ length: uploadingCount }).map((_, index) => (
<SearchTestImagePreviewItem key={index} item={{ type: 'uploading', key: `${index}` }} />
))}
</Flex>
);
};
export default React.memo(SearchTestImagePreviewList);
const SearchTestImagePreviewItem = React.memo(function SearchTestImagePreviewItem({
item,
onRemove
}: {
item: SearchTestImagePreviewListItem;
onRemove?: (key: string) => void;
}) {
return (
<Flex position={'relative'} overflow={'visible'} {...searchTestImageThumbProps}>
{item.type === 'image' ? (
<>
<Box
as={'img'}
src={item.previewUrl}
alt=""
w={'100%'}
h={'100%'}
objectFit={'cover'}
borderRadius={'6.66667px'}
/>
<Box
position={'absolute'}
right={'-7.5px'}
top={'-7.5px'}
w={'16.67px'}
h={'16.67px'}
display={'flex'}
alignItems={'center'}
justifyContent={'center'}
bg={'myGray.400'}
borderRadius={'50%'}
boxShadow={
'0px 6.66667px 6.66667px rgba(19, 51, 107, 0.1), 0px 0px 1.66667px rgba(19, 51, 107, 0.08)'
}
cursor={'pointer'}
onClick={() => onRemove?.(item.key)}
>
<MyIcon name={'common/closeLight'} w={'11.9px'} h={'11.9px'} color={'white'} />
</Box>
</>
) : (
<CircularProgress
value={28}
size={'46.67px'}
thickness={'8px'}
color={'primary.600'}
trackColor={'myGray.250'}
capIsRound
/>
)}
</Flex>
);
});
projects/app/src/pageComponents/dataset/detail/Test/components/TestHistories.tsx 0 → 100644
import React, { useMemo } from 'react';
import { Box, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { formatTimeToChatTime } from '@fastgpt/global/common/string/time';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import ImagePreviewToken from '@/components/core/dataset/ImagePreviewToken';
import {
useSearchTestStore,
type SearchTestStoreItemType
} from '@/web/core/dataset/store/searchTest';
const TestHistories = ({
datasetId,
datasetTestItem,
onSelect,
onClearSelect
}: {
datasetId: string;
datasetTestItem?: SearchTestStoreItemType;
onSelect: (item: SearchTestStoreItemType) => void;
onClearSelect: () => void;
}) => {
const { t } = useTranslation();
const { datasetTestList, delDatasetTestItemById } = useSearchTestStore();
// The store is shared across dataset pages; show only the current dataset's test records.
const testHistories = useMemo(
() => datasetTestList.filter((item) => item.datasetId === datasetId),
[datasetId, datasetTestList]
);
return (
<>
<Flex alignItems={'center'} color={'myGray.900'}>
<Box fontSize={'md'} fontWeight={500}>
{t('common:core.dataset.test.test history')}
</Box>
</Flex>
<Box mt={3} display={'flex'} flexDirection={'column'} gap={2}>
{testHistories.map((item) => (
<Flex
key={item.id}
position={'relative'}
py={2}
px={3}
alignItems={'center'}
borderColor={'borderColor.low'}
borderWidth={'1px'}
borderRadius={'md'}
_hover={{
borderColor: 'primary.300',
boxShadow: '1',
'& .delete': {
display: 'block'
},
'& .time': {
display: 'none'
}
}}
cursor={'pointer'}
fontSize={'sm'}
{...(item.id === datasetTestItem?.id && {
bg: 'primary.50'
})}
onClick={() => onSelect(item)}
>
<Box
flex={1}
mr={2}
wordBreak={'break-all'}
fontWeight={'400'}
display={'flex'}
alignItems={'center'}
flexWrap={'wrap'}
gap={1}
>
{!!item.text && <Box as={'span'}>{item.text}</Box>}
<ImagePreviewToken
images={item.queryImageRefs || []}
datasetId={datasetId}
containerProps={{
as: 'span',
display: 'inline-flex',
gap: 1
}}
tokenProps={{
px: 0,
py: 0,
border: 'none',
bg: 'transparent',
color: 'inherit'
}}
/>
</Box>
<Box className="time" flex={'0 0 auto'} fontSize={'xs'} color={'myGray.500'}>
{t(formatTimeToChatTime(item.time) as any).replace('#', ':')}
</Box>
<MyTooltip label={t('common:core.dataset.test.delete test history')}>
<Box className="delete" display={'none'} w={'0.8rem'} h={'0.8rem'} ml={1}>
<MyIcon
name={'delete'}
w={'0.8rem'}
_hover={{ color: 'red.600' }}
onClick={(e) => {
e.stopPropagation();
delDatasetTestItemById(item.id);
if (datasetTestItem?.id === item.id) {
onClearSelect();
}
}}
/>
</Box>
</MyTooltip>
</Flex>
))}
</Box>
</>
);
};
export default React.memo(TestHistories);
projects/app/src/pageComponents/dataset/detail/Test/components/TestInputPanel.tsx 0 → 100644
import React from 'react';
import { Box, Button, Flex, Textarea } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { SEARCH_TEST_IMAGE_UPLOAD_ENABLED } from '../constants';
import SearchTestImagePreviewList from './SearchTestImagePreviewList';
import type { SearchTestFormType, SearchTestImageRef } from '../type';
import type { UseFormRegister } from 'react-hook-form';
const TestInputPanel = ({
canSubmit,
canUseImageSearch,
datasetMaxToken,
isLoading,
onOpenImageSelector,
onOpenSelectMode,
onRemoveImage,
onSubmit,
queryImageRefs,
register,
showSearchTestImageEntry,
uploadingImageCount
}: {
canSubmit: boolean;
canUseImageSearch: boolean;
datasetMaxToken?: number;
isLoading: boolean;
onOpenImageSelector: () => void;
onOpenSelectMode: () => void;
onRemoveImage: (key: string) => void;
onSubmit: () => void;
queryImageRefs: SearchTestImageRef[];
register: UseFormRegister<SearchTestFormType>;
showSearchTestImageEntry: boolean;
uploadingImageCount: number;
}) => {
const { t } = useTranslation();
return (
<Box
display={'flex'}
flexDirection={'column'}
alignItems={'flex-start'}
gap={3}
alignSelf={'stretch'}
>
<Flex alignItems={'center'} alignSelf={'stretch'}>
<Box flex={1} fontWeight={500} color={'myGray.900'}>
{t('common:core.dataset.test.input_title')}
</Box>
<Button
variant={'whitePrimary'}
leftIcon={<MyIcon name={'common/settingLight'} w={'14px'} />}
size={'sm'}
fontWeight={500}
onClick={onOpenSelectMode}
>
{t('common:core.dataset.test.search_config')}
</Button>
</Flex>
<Box
border={'1px solid'}
borderColor={'borderColor.low'}
p={3}
borderRadius={'6px'}
minH={'220px'}
display={'flex'}
flexDirection={'column'}
bg={'white'}
alignSelf={'stretch'}
position={'relative'}
>
{showSearchTestImageEntry && (
<SearchTestImagePreviewList
images={queryImageRefs}
uploadingCount={uploadingImageCount}
onRemove={onRemoveImage}
/>
)}
<Textarea
flex={1}
minH={'140px'}
resize={'none'}
variant={'unstyled'}
py={0}
fontSize={'sm'}
lineHeight={'20px'}
color={'myGray.900'}
_placeholder={{
color: 'myGray.400'
}}
maxLength={datasetMaxToken}
placeholder={t('common:core.dataset.test.Test Text Placeholder')}
{...register('inputText')}
/>
{SEARCH_TEST_IMAGE_UPLOAD_ENABLED && (
<MyTooltip
label={canUseImageSearch ? '' : t('common:core.dataset.test.image_search_disabled_tip')}
>
<Box position={'absolute'} left={'12px'} bottom={'8px'}>
<Box
as={'button'}
w={'24px'}
h={'24px'}
p={0}
display={'flex'}
alignItems={'center'}
justifyContent={'center'}
bg={'transparent'}
border={'none'}
boxShadow={'none'}
_hover={{
bg: 'transparent'
}}
_active={{
bg: 'transparent'
}}
_disabled={{
bg: 'transparent',
opacity: 0.5,
cursor: 'not-allowed'
}}
disabled={!canUseImageSearch}
onClick={onOpenImageSelector}
aria-label={t('common:core.dataset.test.upload_image')}
>
<MyIcon name={'image'} w={'20px'} h={'20px'} color={'myGray.500'} flexShrink={0} />
</Box>
</Box>
</MyTooltip>
)}
</Box>
<Button
w={'100%'}
isLoading={isLoading}
isDisabled={!canSubmit || uploadingImageCount > 0}
onClick={onSubmit}
>
{t('common:core.dataset.test.Test')}
</Button>
</Box>
);
};
export default React.memo(TestInputPanel);
projects/app/src/pageComponents/dataset/detail/Test/components/TestResults.tsx 0 → 100644
import React from 'react';
import { Box, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import QuoteItem from '@/components/core/dataset/QuoteItem';
import SearchParamsTip from '@/components/core/dataset/SearchParamsTip';
import type { SearchTestStoreItemType } from '@/web/core/dataset/store/searchTest';
const TestResults = ({ datasetTestItem }: { datasetTestItem?: SearchTestStoreItemType }) => {
const { t } = useTranslation();
if (!datasetTestItem?.results || datasetTestItem.results.length === 0) {
return (
<EmptyTip text={t('common:core.dataset.test.test result placeholder')} mt={[10, '20vh']} />
);
}
return (
<>
<Flex fontSize={'md'} color={'myGray.900'} alignItems={'center'} fontWeight={500}>
{t('common:core.dataset.test.Test params')}
</Flex>
<Box mt={3}>
<SearchParamsTip
searchMode={datasetTestItem.searchMode}
similarity={datasetTestItem.similarity}
limit={datasetTestItem.limit}
usingReRank={datasetTestItem.usingReRank}
usingExtensionQuery={!!datasetTestItem.queryExtensionModel}
queryExtensionModel={datasetTestItem.queryExtensionModel}
/>
</Box>
<Flex mt={5} mb={3} alignItems={'center'}>
<Flex fontSize={'md'} color={'myGray.900'} alignItems={'center'} fontWeight={500}>
{t('common:core.dataset.test.Test Result')}
</Flex>
<QuestionTip ml={1} label={t('common:core.dataset.test.test result tip')} />
<Box ml={2}>({datasetTestItem.duration})</Box>
</Flex>
<Box mt={1} gap={4}>
{datasetTestItem.results.map((item) => (
<Box key={item.id} p={3} borderRadius={'lg'} bg={'myGray.100'} _notLast={{ mb: 2 }}>
<QuoteItem quoteItem={item} canDownloadSource canEditData />
</Box>
))}
</Box>
</>
);
};
export default React.memo(TestResults);
projects/app/src/pageComponents/dataset/detail/Test/constants.ts 0 → 100644
import { imageFileType } from '@fastgpt/global/common/file/constants';
// Keep the disabled upload path in place so the search test flow can enable images by flag only.
export const SEARCH_TEST_IMAGE_UPLOAD_ENABLED = false;
export const MAX_SEARCH_TEST_IMAGE_COUNT = 10;
// Derive accepted extensions from the shared upload accept string to avoid frontend/backend drift.
export const IMAGE_EXTENSION_SET = new Set(
imageFileType
.split(',')
.map((item) => item.trim().replace('.', '').toLowerCase())
.filter(Boolean)
);
export const searchTestImageThumbProps = {
w: '80px',
h: '80px',
alignItems: 'center',
justifyContent: 'center',
bg: 'white',
border: '1.07143px solid',
borderColor: 'borderColor.low',
borderRadius: '8px',
boxShadow:
'0px 4.28571px 10.7143px rgba(19, 51, 107, 0.08), 0px 0px 1.07143px rgba(19, 51, 107, 0.08)'
} as const;
projects/app/src/pageComponents/dataset/detail/Test/hooks/useDatasetSearchTest.ts 0 → 100644
import { useState } from 'react';
import { useForm, useWatch } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import { postSearchText } from '@/web/core/dataset/api';
import {
useSearchTestStore,
type SearchTestStoreItemType
} from '@/web/core/dataset/store/searchTest';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import type { SearchDatasetTestResponse } from '@fastgpt/global/openapi/core/dataset/api';
import type { SearchTestFormType, SearchTestImageRef } from '../type';
export const useDatasetSearchTest = ({
datasetId,
queryImageRefs,
defaultModels
}: {
datasetId: string;
queryImageRefs: SearchTestImageRef[];
defaultModels: {
rerank?: {
model?: string;
};
llm?: {
model?: string;
};
};
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { pushDatasetTestItem } = useSearchTestStore();
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
const { control, getValues, setValue, register, handleSubmit } = useForm<SearchTestFormType>({
defaultValues: {
inputText: '',
searchParams: {
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false,
rerankModel: defaultModels?.rerank?.model,
rerankWeight: 0.5,
limit: 5000,
similarity: 0,
datasetSearchUsingExtensionQuery: false,
datasetSearchExtensionModel: defaultModels.llm?.model,
datasetSearchExtensionBg: ''
}
}
});
const inputText = useWatch({ control, name: 'inputText' });
const searchParams = getValues('searchParams');
const { runAsync: onTextTest, loading: textTestIsLoading } = useRequest(
({ inputText, searchParams }: SearchTestFormType) =>
postSearchText({
datasetId,
text: inputText.trim(),
// The backend resolves uploaded images by object key; previews stay client-side only.
queryImageUrls: queryImageRefs.map((item) => item.key),
...searchParams
}),
{
onSuccess(res: SearchDatasetTestResponse) {
if (!res || res.list.length === 0) {
return toast({
status: 'warning',
title: t('common:dataset.test.noResult')
});
}
const testItem: SearchTestStoreItemType = {
id: getNanoid(),
datasetId,
text: getValues('inputText').trim(),
time: new Date(),
results: res.list,
// Histories only need stable image keys; preview urls are regenerated by ImagePreviewToken.
queryImageRefs:
queryImageRefs.length > 0 ? queryImageRefs.map(({ key }) => ({ key })) : undefined,
duration: res.duration,
searchMode: res.searchMode,
usingReRank: res.usingReRank,
limit: res.limit,
similarity: res.similarity,
queryExtensionModel: res.queryExtensionModel
};
pushDatasetTestItem(testItem);
setDatasetTestItem(testItem);
}
}
);
return {
setDatasetTestItem,
currentDatasetTestItem: datasetTestItem?.datasetId === datasetId ? datasetTestItem : undefined,
inputText,
searchParams,
setValue,
register,
onSubmit: handleSubmit((data) => onTextTest(data)),
textTestIsLoading
};
};
projects/app/src/pageComponents/dataset/detail/Test/hooks/useSearchTestImages.tsx 0 → 100644
import { useState } from 'react';
import { useTranslation } from 'next-i18next';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { getUploadSearchTestImagePresignedUrl } from '@/web/core/dataset/api/file';
import { useUserStore } from '@/web/support/user/useUserStore';
import { imageFileType } from '@fastgpt/global/common/file/constants';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { putFileToS3 } from '@fastgpt/web/common/file/utils';
import { useToast } from '@fastgpt/web/hooks/useToast';
import {
IMAGE_EXTENSION_SET,
MAX_SEARCH_TEST_IMAGE_COUNT,
SEARCH_TEST_IMAGE_UPLOAD_ENABLED
} from '../constants';
import type { SearchTestImageRef } from '../type';
export const useSearchTestImages = ({
datasetId,
canUseImageSearch,
uploadFileMaxSize
}: {
datasetId: string;
canUseImageSearch: boolean;
uploadFileMaxSize?: number;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { teamPlanStatus, initTeamPlanStatus } = useUserStore();
const { File: ImageFileSelector, onOpen: onOpenImageSelector } = useSelectFile({
fileType: imageFileType,
multiple: true,
maxCount: MAX_SEARCH_TEST_IMAGE_COUNT
});
const [queryImageRefs, setQueryImageRefs] = useState<SearchTestImageRef[]>([]);
const [uploadingImageCount, setUploadingImageCount] = useState(0);
// Feature flag lets the UI stay wired while image search upload is being rolled out.
const showSearchTestImageEntry = SEARCH_TEST_IMAGE_UPLOAD_ENABLED && canUseImageSearch;
const onSelectFile = async (files: File[]) => {
const imageFiles = files.filter((file) => {
const extension = file.name.split('.').pop()?.toLowerCase();
return !!extension && IMAGE_EXTENSION_SET.has(extension);
});
if (imageFiles.length < files.length) {
toast({
status: 'warning',
title: t('chat:unsupported_file_type')
});
}
// Prefer live team limits; fall back to frontend config so validation still works offline.
const planStatus =
teamPlanStatus ||
(await initTeamPlanStatus()
.then(() => useUserStore.getState().teamPlanStatus)
.catch(() => undefined));
const maxImageSize =
(planStatus?.standard?.maxUploadFileSize ?? uploadFileMaxSize ?? 500) * 1024 * 1024;
const validImageFiles = imageFiles.filter((file) => file.size <= maxImageSize);
if (validImageFiles.length < imageFiles.length) {
toast({
status: 'warning',
title: t('file:some_file_size_exceeds_limit', {
maxSize: formatFileSize(maxImageSize)
})
});
}
if (queryImageRefs.length + validImageFiles.length > MAX_SEARCH_TEST_IMAGE_COUNT) {
toast({
status: 'warning',
title: t('common:core.dataset.test.max_images_tip')
});
}
// Trim the selected batch instead of rejecting all files when the max count is exceeded.
const uploadFiles = validImageFiles.slice(
0,
Math.max(MAX_SEARCH_TEST_IMAGE_COUNT - queryImageRefs.length, 0)
);
if (uploadFiles.length === 0) return;
setUploadingImageCount(uploadFiles.length);
try {
const uploadedImages = await Promise.all(
uploadFiles.map(async (file) => {
const { url, key, headers, maxSize, previewUrl } =
await getUploadSearchTestImagePresignedUrl({
datasetId,
filename: file.name
});
await putFileToS3({
url,
headers,
file,
maxSize,
t
});
return { key, previewUrl };
})
);
setQueryImageRefs((state) => [...state, ...uploadedImages]);
} catch {
toast({
status: 'warning',
title: t('common:upload_file_error')
});
} finally {
setUploadingImageCount(0);
}
};
return {
ImageFileSelector,
queryImageRefs,
uploadingImageCount,
showSearchTestImageEntry,
onOpenImageSelector,
onSelectFile,
removeImage: (key: string) =>
setQueryImageRefs((state) => state.filter((item) => item.key !== key))
};
};
projects/app/src/pageComponents/dataset/detail/Test/index.tsx 0 → 100644
import React from 'react';
import dynamic from 'next/dynamic';
import { Box, useDisclosure } from '@chakra-ui/react';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { SEARCH_TEST_IMAGE_UPLOAD_ENABLED } from './constants';
import TestInputPanel from './components/TestInputPanel';
import TestHistories from './components/TestHistories';
import TestResults from './components/TestResults';
import { useDatasetSearchTest } from './hooks/useDatasetSearchTest';
import { useSearchTestImages } from './hooks/useSearchTestImages';
const DatasetParamsModal = dynamic(() => import('@/components/core/app/DatasetParamsModal'));
const Test = ({ datasetId }: { datasetId: string }) => {
const { defaultModels, feConfigs } = useSystemStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
// Image search is only meaningful when the dataset has a vision vector model or VLM configured.
const canUseImageSearch = !!datasetDetail.vectorModel?.vision || !!datasetDetail.vlmModel;
const {
ImageFileSelector,
queryImageRefs,
uploadingImageCount,
showSearchTestImageEntry,
onOpenImageSelector,
onSelectFile,
removeImage
} = useSearchTestImages({
datasetId,
canUseImageSearch,
uploadFileMaxSize: feConfigs?.uploadFileMaxSize
});
const {
currentDatasetTestItem,
inputText,
searchParams,
setValue,
register,
onSubmit,
textTestIsLoading,
setDatasetTestItem
} = useDatasetSearchTest({
datasetId,
queryImageRefs: showSearchTestImageEntry ? queryImageRefs : [],
defaultModels
});
const {
isOpen: isOpenSelectMode,
onOpen: onOpenSelectMode,
onClose: onCloseSelectMode
} = useDisclosure();
// Text and image can both be search inputs; keep image-only test available for multimodal datasets.
const canSubmit = !!inputText?.trim() || (showSearchTestImageEntry && queryImageRefs.length > 0);
return (
<Box h={'100%'} display={['block', 'flex']}>
<Box
h={['auto', '100%']}
display={['block', 'flex']}
flexDirection={'column'}
flex={['unset', '0 0 468px']}
w={['100%', '468px']}
p={4}
gap={6}
borderRightWidth={['0', '1px']}
borderRightStyle={'solid'}
borderRightColor={'borderColor.low'}
>
<TestInputPanel
canSubmit={canSubmit}
canUseImageSearch={canUseImageSearch}
datasetMaxToken={datasetDetail.vectorModel?.maxToken}
isLoading={textTestIsLoading}
onOpenImageSelector={onOpenImageSelector}
onOpenSelectMode={onOpenSelectMode}
onRemoveImage={removeImage}
onSubmit={onSubmit}
queryImageRefs={queryImageRefs}
register={register}
showSearchTestImageEntry={showSearchTestImageEntry}
uploadingImageCount={uploadingImageCount}
/>
<Box overflow={'overlay'} display={['none', 'block']}>
<TestHistories
datasetId={datasetId}
datasetTestItem={currentDatasetTestItem}
onSelect={setDatasetTestItem}
onClearSelect={() => setDatasetTestItem(undefined)}
/>
</Box>
</Box>
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={'1 0 0'} bg={'white'}>
<TestResults datasetTestItem={currentDatasetTestItem} />
</Box>
{isOpenSelectMode && (
<DatasetParamsModal
{...searchParams}
maxTokens={20000}
onClose={onCloseSelectMode}
onSuccess={(e) => {
setValue('searchParams', {
...searchParams,
...e
});
}}
/>
)}
{SEARCH_TEST_IMAGE_UPLOAD_ENABLED && <ImageFileSelector onSelect={onSelectFile} />}
</Box>
);
};
export default React.memo(Test);
projects/app/src/pageComponents/dataset/detail/Test/type.ts 0 → 100644
import type { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
export type SearchTestFormType = {
inputText: string;
searchParams: {
searchMode: DatasetSearchModeEnum;
embeddingWeight?: number;
usingReRank?: boolean;
rerankModel?: string;
rerankWeight?: number;
similarity?: number;
limit?: number;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
};
};
export type SearchTestImageRef = {
key: string;
previewUrl: string;
};
projects/app/src/pageComponents/dataset/detail/components/InputDataModal/DataInputPanel.tsx 0 → 100644
import React from 'react';
import { Box, Button, Flex, Spinner, Textarea, type TextareaProps } from '@chakra-ui/react';
import type { UseFormRegister } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyImage from '@/components/MyImage/index';
import { TabEnum, type InputDataType } from './useInputDataModal';
const dataTextareaProps: TextareaProps = {
resize: 'both',
flex: '1 0 0',
bg: 'white',
borderRadius: '6px',
border: '1px solid',
borderColor: 'borderColor.low',
p: '8px 12px',
color: 'myGray.900',
_focus: {
borderColor: 'primary.500',
boxShadow: 'focus',
bg: 'white'
},
sx: {
'&::-webkit-scrollbar-thumb': {
background: 'var(--chakra-colors-myGray-150) !important',
transition: 'background 1s',
marginLeft: '5px'
},
'&::-webkit-scrollbar-thumb:hover': {
background: 'var(--chakra-colors-myGray-250) !important'
}
}
};
const getDataTextareaProps = (canWrite: boolean): TextareaProps => ({
...dataTextareaProps,
isReadOnly: !canWrite,
cursor: canWrite ? 'text' : 'default',
_readOnly: {
bg: 'myGray.25',
color: 'myGray.600'
}
});
const DataInputPanel = ({
canWrite,
currentTab,
imagePreivewUrl,
isImporting,
isUpdating,
isIndexEditing,
register,
submitData
}: {
canWrite: boolean;
currentTab?: TabEnum;
imagePreivewUrl?: string;
isImporting: boolean;
isUpdating: boolean;
isIndexEditing: boolean;
register: UseFormRegister<InputDataType>;
submitData: () => void;
}) => {
const { t } = useTranslation();
const textareaProps = getDataTextareaProps(canWrite);
const isSubmitting = isImporting || isUpdating;
const isSubmitDisabled = !canWrite || isIndexEditing;
const iconColor = isSubmitDisabled ? 'myGray.400' : 'primary.600';
return (
<Flex
flexDir={'column'}
gap={'8px'}
flex={'1 0 0'}
w={['100%', 0]}
overflow={['unset', 'auto']}
>
<Flex
flexDir={'column'}
flex={currentTab === TabEnum.image ? '0 0 201px' : '1 0 0'}
h={currentTab === TabEnum.image ? '201px' : 0}
minH={0}
gap={'8px'}
>
{currentTab === TabEnum.image && (
<>
<FormLabel required h={'32px'} py={'6px'}>
{t('file:image')}
</FormLabel>
<Box flex={'1 0 0'} h={0} w="100%">
<Box
height="100%"
position="relative"
border={'1px solid'}
borderColor={'borderColor.low'}
borderRadius={'6px'}
bg={'myGray.25'}
p={'8px'}
>
<MyImage
src={imagePreivewUrl}
h="100%"
w="100%"
objectFit="cover"
borderRadius={'2px'}
alt={t('file:Image_Preview')}
/>
</Box>
</Box>
</>
)}
{(currentTab === TabEnum.chunk || currentTab === TabEnum.qa) && (
<>
<FormLabel required h={'20px'}>
{currentTab === TabEnum.chunk
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
<Textarea
{...textareaProps}
{...register(`q`, {
required: true
})}
/>
</>
)}
</Flex>
{currentTab === TabEnum.qa && (
<Flex flexDir={'column'} flex={'1 0 0'} h={0} minH={0} gap={'8px'}>
<FormLabel required h={'20px'}>
{t('common:dataset_data_input_a')}
</FormLabel>
<Textarea {...textareaProps} {...register('a', { required: true })} />
</Flex>
)}
{currentTab === TabEnum.image && (
<Flex flexDir={'column'} flex={'1 0 0'} h={0} minH={0} gap={'8px'}>
<FormLabel required h={'32px'} py={'6px'}>
{t('file:image_description')}
</FormLabel>
<Textarea
{...textareaProps}
placeholder={t('file:image_description_tip')}
{...register('q', {
required: true
})}
/>
</Flex>
)}
<Button
h={'32px'}
minH={'32px'}
w={'100%'}
bg={'myGray.150'}
color={'primary.700'}
borderRadius={'6px'}
fontSize={'12px'}
lineHeight={'16px'}
fontWeight={'500'}
letterSpacing={'0.5px'}
_hover={{ bg: 'myGray.200' }}
_disabled={{
bg: 'myGray.100',
color: 'myGray.400',
cursor: 'not-allowed',
opacity: 1,
_hover: {
bg: 'myGray.100'
}
}}
rightIcon={
isSubmitting ? (
<Spinner w={'16px'} h={'16px'} color={iconColor} />
) : (
<MyIcon name={'common/rightArrowLight'} w={'16px'} color={iconColor} />
)
}
isDisabled={isSubmitDisabled}
aria-busy={isSubmitting}
onClick={() => {
if (isSubmitting || isSubmitDisabled) return;
submitData();
}}
>
{t('dataset:generate_index')}
</Button>
</Flex>
);
};
export default React.memo(DataInputPanel);
projects/app/src/pageComponents/dataset/detail/components/InputDataModal/IndexInputPanel.tsx 0 → 100644
import React, { useCallback, useEffect, useRef } from 'react';
import { Box, Button, Flex, Textarea, VStack } from '@chakra-ui/react';
import type { UseFieldArrayReturn, UseFormRegister } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import DeleteIcon from '@fastgpt/web/components/common/Icon/delete';
import MyBox from '@fastgpt/web/components/common/MyBox';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyIconButton from '@fastgpt/web/components/common/Icon/button';
import {
DatasetDataIndexTypeEnum,
getDatasetIndexMapData
} from '@fastgpt/global/core/dataset/data/constants';
import type { InputDataType } from './useInputDataModal';
type IndexField = UseFieldArrayReturn<InputDataType, 'indexes'>['fields'][number];
const IndexInputPanel = ({
canWrite,
deletingIndexClientId,
focusIndexClientId,
indexes,
isDeletingIndex,
maxToken,
register,
watchedIndexes,
onAddIndex,
onDeleteIndex,
onSaveIndex,
onAutoFocusIndex,
onIndexFocus,
onIndexBlur,
onDeleteIntent,
updateIndexFold
}: {
canWrite: boolean;
deletingIndexClientId?: string;
focusIndexClientId?: string;
indexes: IndexField[];
isDeletingIndex: boolean;
maxToken: number;
register: UseFormRegister<InputDataType>;
watchedIndexes?: InputDataType['indexes'];
onAddIndex: () => void;
onDeleteIndex: (clientId: string) => void;
onSaveIndex: (clientId: string) => void;
onAutoFocusIndex: (clientId: string) => void;
onIndexFocus: (clientId: string) => void;
onIndexBlur: (clientId: string) => void;
onDeleteIntent: (clientId: string) => void;
updateIndexFold: (clientId: string, fold: boolean) => void;
}) => {
const { t } = useTranslation();
return (
<Flex flexDir={'column'} flex={'1 0 0'} w={['100%', 0]} minH={0}>
<Flex
alignItems={'center'}
justifyContent={'space-between'}
h={'20px'}
mb={'8px'}
flexShrink={0}
>
<FormLabel color={'myGray.900'} fontSize={'14px'} lineHeight={'20px'}>
{t('common:dataset.data.edit.Index', {
amount: indexes.length
})}
</FormLabel>
<Button
variant={'whiteBase'}
size={'sm'}
h={'30px'}
px={'14px'}
py={'7px'}
borderRadius={'6px'}
isDisabled={!canWrite}
onClick={onAddIndex}
>
<Flex alignItems={'center'} fontSize={'12px'} lineHeight={'16px'} color={'myGray.600'}>
<MyIcon name={'common/addLight'} w={'1rem'} mr={'6px'} />
{t('common:add_new')}
</Flex>
</Button>
</Flex>
<Box flex={'1 0 0'} h={0} minH={0} overflow={['unset', 'auto']}>
<VStack spacing={'8px'} alignItems={'stretch'}>
{indexes?.map((field, i) => {
const index = watchedIndexes?.[i] || field;
const data = getDatasetIndexMapData(index.type);
const canFoldIndex = indexes.length > 1;
const hasIndexDataId = !!index.dataId;
const isDeletingCurrentIndex = deletingIndexClientId === index.clientId;
const isDefaultIndex = index.type === DatasetDataIndexTypeEnum.default;
const canDeleteIndex =
canWrite && !isDefaultIndex && hasIndexDataId && !isDeletingCurrentIndex;
const canToggleFold = canFoldIndex && !isDeletingCurrentIndex;
return (
<MyBox
key={field.clientId}
isLoading={isDeletingCurrentIndex}
p={'16px'}
borderRadius={'8px'}
border={'1px solid'}
borderColor={'borderColor.low'}
bg={'myGray.25'}
w={'100%'}
minH={'104px'}
// MyBox renders loading as an overlay; keep the textarea mounted to avoid height jumps.
pointerEvents={isDeletingCurrentIndex ? 'none' : 'auto'}
_hover={{
'& .delete': {
display: 'block'
}
}}
>
<Flex mb={'8px'} alignItems={'center'} h={'24px'}>
<FormLabel
flex={'1 0 0'}
color={'myGray.900'}
fontSize={'14px'}
lineHeight={'20px'}
>
{t(data.label)}
</FormLabel>
{canDeleteIndex && (
<Flex className={'delete'} display={'none'} borderRight={'base'} pr={3} mr={2}>
<DeleteIcon
onMouseDown={(e) => {
e.preventDefault();
onDeleteIntent(index.clientId);
}}
onClick={() => {
if (isDeletingIndex) return;
onDeleteIndex(index.clientId);
}}
/>
</Flex>
)}
{canToggleFold && (
<MyIconButton
icon={index.fold ? 'core/chat/chevronDown' : 'core/chat/chevronUp'}
w={'24px'}
h={'24px'}
color={'myGray.500'}
hoverBg={'transparent'}
onClick={() => {
updateIndexFold(index.clientId, !index.fold);
}}
/>
)}
</Flex>
<DataIndexTextArea
disabled={!canWrite || isDefaultIndex}
canClickMark={hasIndexDataId}
autoFocus={focusIndexClientId === index.clientId}
index={i}
value={index.text}
isFolder={index.fold && canFoldIndex}
maxToken={maxToken}
register={register}
onFocus={() => {
onIndexFocus(index.clientId);
updateIndexFold(index.clientId, false);
}}
onAutoFocus={() => {
onAutoFocusIndex(index.clientId);
}}
onBlur={() => {
onIndexBlur(index.clientId);
if (!canWrite) return;
onSaveIndex(index.clientId);
}}
/>
</MyBox>
);
})}
</VStack>
</Box>
</Flex>
);
};
export default React.memo(IndexInputPanel);
const textareaMinH = '40px';
const DataIndexTextArea = ({
value,
index,
maxToken,
register,
disabled,
canClickMark,
autoFocus,
isFolder,
onFocus,
onAutoFocus,
onBlur: onSaveBlur
}: {
value: string;
index: number;
maxToken: number;
register: UseFormRegister<InputDataType>;
disabled?: boolean;
canClickMark?: boolean;
autoFocus?: boolean;
isFolder: boolean;
onFocus: () => void;
onAutoFocus?: () => void;
onBlur: () => void;
}) => {
const { t } = useTranslation();
const TextareaDom = useRef<HTMLTextAreaElement | null>(null);
const {
ref: TextareaRef,
required,
name,
onChange: onTextChange,
onBlur
} = register(`indexes.${index}.text`, { required: true });
useEffect(() => {
if (TextareaDom.current) {
TextareaDom.current.style.height = textareaMinH;
TextareaDom.current.style.height = `${TextareaDom.current.scrollHeight + 5}px`;
}
}, []);
useEffect(() => {
if (!autoFocus || disabled || !TextareaDom.current) return;
// New rows are prepended before the textarea ref is ready, so focus on the next tick.
const timer = window.setTimeout(() => {
TextareaDom.current?.focus();
onFocus();
onAutoFocus?.();
});
return () => {
window.clearTimeout(timer);
};
}, [autoFocus, disabled, onAutoFocus, onFocus]);
const autoHeight = useCallback((e: React.ChangeEvent<HTMLTextAreaElement>) => {
if (e.target) {
e.target.style.height = textareaMinH;
e.target.style.height = `${e.target.scrollHeight + 5}px`;
}
}, []);
const handleClickFoldMask = () => {
TextareaDom?.current?.focus();
onFocus();
};
return (
<Box
pos={'relative'}
{...(isFolder
? {
maxH: '40px',
overflow: 'hidden'
}
: {
maxH: 'auto'
})}
>
{disabled ? (
<Box fontSize={'sm'} color={'myGray.600'} letterSpacing={'0.004em'} whiteSpace={'pre-wrap'}>
{value}
</Box>
) : (
<Textarea
maxLength={maxToken}
borderColor={'transparent'}
minH={'32px'}
px={0}
pt={0}
isRequired={required}
whiteSpace={'pre-wrap'}
resize={'none'}
fontSize={'sm'}
color={'myGray.500'}
letterSpacing={'0.004em'}
_focus={{
px: 3,
py: 1,
borderColor: 'primary.500',
boxShadow: 'focus',
bg: 'white'
}}
sx={{
'&::-webkit-scrollbar-thumb': {
background: 'var(--chakra-colors-myGray-150) !important',
transition: 'background 1s',
marginLeft: '5px'
},
'&::-webkit-scrollbar-thumb:hover': {
background: 'var(--chakra-colors-myGray-250) !important'
}
}}
placeholder={t('common:dataset.data.Index Placeholder')}
ref={(e) => {
if (e) TextareaDom.current = e;
TextareaRef(e);
}}
required
name={name}
onChange={(e) => {
autoHeight(e);
onTextChange(e);
}}
onFocus={autoHeight}
onBlur={(e) => {
onBlur(e);
onSaveBlur();
}}
/>
)}
{isFolder && (
<Box
pos={'absolute'}
bottom={0}
left={0}
right={0}
top={0}
bg={
'linear-gradient(182deg, rgba(251, 251, 252, 0.00) 1.76%, var(--chakra-colors-myGray-25) 84.07%)'
}
borderRadius={'6px'}
{...(disabled || !canClickMark
? {}
: {
// Unsaved rows can still be edited directly, but folded overlay click needs a backend id.
cursor: 'pointer',
onClick: handleClickFoldMask
})}
/>
)}
</Box>
);
};
projects/app/src/pageComponents/dataset/detail/components/InputDataModal/index.tsx 0 → 100644
import React from 'react';
import { Box, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import MyModal from '@fastgpt/web/components/v2/common/MyModal';
import MyBox from '@fastgpt/web/components/common/MyBox';
import DataInputPanel from './DataInputPanel';
import IndexInputPanel from './IndexInputPanel';
import { TabEnum, useInputDataModal, type InputDataType } from './useInputDataModal';
const InputDataModal = ({
collectionId,
dataId,
defaultValue,
onClose,
onSuccess
}: {
collectionId: string;
dataId?: string;
defaultValue?: { q?: string; a?: string; imagePreivewUrl?: string };
onClose: () => void;
onSuccess: (data: InputDataType & { dataId: string }) => void;
}) => {
const { t } = useTranslation();
const {
collection,
currentTab,
deletingIndexClientId,
editingIndexClientId,
focusIndexClientId,
imagePreivewUrl,
indexes,
initLoading,
isDeletingIndex,
isImporting,
isUpdating,
maxToken,
register,
showTabs,
submitData,
watchedIndexes,
onDeleteIndex,
onSaveIndex,
prependCustomIndex,
setCurrentTab,
clearFocusIndexClientId,
markEditingIndex,
clearEditingIndex,
markDeletingIndex,
updateIndexFold
} = useInputDataModal({
collectionId,
dataId,
defaultValue,
onSuccess
});
return (
<MyModal
isOpen={true}
isCentered
w={['calc(100vw - 32px)', '800px']}
onClose={onClose}
closeOnOverlayClick={false}
maxW={['calc(100vw - 32px)', '800px']}
h={['auto', currentTab === TabEnum.image ? '584px' : '620px']}
maxH={['90vh', 'calc(100vh - 48px)']}
borderRadius={'10px'}
bg={'white'}
boxShadow={'3.5'}
contentPx={[5, '32px']}
contentPy={[5, '32px']}
headerPx={0}
title={
<Box
className={'textEllipsis'}
wordBreak={'break-all'}
fontSize={['xl', '20px']}
lineHeight={['28px', '26px']}
maxW={['calc(100vw - 96px)', '680px']}
fontWeight={'500'}
letterSpacing={'0.15px'}
color={'black'}
whiteSpace={'nowrap'}
overflow={'hidden'}
textOverflow={'ellipsis'}
>
{collection.sourceName || t('common:unknow_source')}
</Box>
}
>
<MyBox
display={'flex'}
flexDir={'column'}
isLoading={initLoading}
h={'100%'}
minH={['300px', showTabs ? '506px' : '450px']}
overflow={'hidden'}
>
{!initLoading && (
<Flex
flexDir={'column'}
gap={'24px'}
w={'100%'}
h={['auto', showTabs ? '506px' : '450px']}
minH={0}
>
{showTabs && (
<Flex
h={'32px'}
gap={'16px'}
borderBottom={'1px solid'}
borderColor={'borderColor.low'}
flexShrink={0}
>
{[
{ label: t('common:dataset_data_input_chunk'), value: TabEnum.chunk },
{ label: t('common:dataset_data_input_qa'), value: TabEnum.qa }
].map((item) => {
const isActive = currentTab === item.value;
return (
<Flex
key={item.value}
alignItems={'center'}
justifyContent={'center'}
h={'32px'}
px={'4px'}
borderBottom={'1.5px solid'}
borderColor={isActive ? 'primary.600' : 'transparent'}
color={isActive ? 'primary.700' : 'myGray.500'}
fontSize={'16px'}
lineHeight={'24px'}
fontWeight={'500'}
letterSpacing={'0.15px'}
cursor={'pointer'}
onClick={() => setCurrentTab(item.value)}
>
{item.label}
</Flex>
);
})}
</Flex>
)}
<Flex
flex={'1 0 0'}
h={['auto', '450px']}
gap={'32px'}
flexDir={['column', 'row']}
minH={0}
>
<DataInputPanel
canWrite={collection.permission.hasWritePer}
currentTab={currentTab}
imagePreivewUrl={imagePreivewUrl}
isImporting={isImporting}
isUpdating={isUpdating}
isIndexEditing={!!editingIndexClientId}
register={register}
submitData={submitData}
/>
<IndexInputPanel
canWrite={collection.permission.hasWritePer}
deletingIndexClientId={deletingIndexClientId}
focusIndexClientId={focusIndexClientId}
indexes={indexes}
isDeletingIndex={isDeletingIndex}
maxToken={maxToken}
register={register}
watchedIndexes={watchedIndexes}
onAddIndex={prependCustomIndex}
onDeleteIndex={onDeleteIndex}
onSaveIndex={onSaveIndex}
onAutoFocusIndex={clearFocusIndexClientId}
onIndexFocus={markEditingIndex}
onIndexBlur={clearEditingIndex}
onDeleteIntent={markDeletingIndex}
updateIndexFold={updateIndexFold}
/>
</Flex>
</Flex>
)}
</MyBox>
</MyModal>
);
};
export default React.memo(InputDataModal);
projects/app/src/pageComponents/dataset/detail/components/InputDataModal/useInputDataModal.ts 0 → 100644
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useFieldArray, useForm, useWatch } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import { getDatasetCollectionById } from '@/web/core/dataset/api/collection';
import {
postInsertData2Dataset,
getDatasetDataItemById,
createDatasetDataIndex,
deleteDatasetDataIndex,
putDatasetDataById,
updateDatasetDataIndex
} from '@/web/core/dataset/api/data';
import { defaultCollectionDetail } from '@/web/core/dataset/constants';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useToast } from '@fastgpt/web/hooks/useToast';
import type { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
export type InputDataType = {
q: string;
a: string;
imagePreivewUrl?: string;
indexes: (Omit<DatasetDataIndexItemType, 'dataId'> & {
// Backend index data id. New client-side rows do not have it until create succeeds.
dataId?: string;
// Stable client identity for optimistic rows and rapid edits.
clientId: string;
fold: boolean;
})[];
};
export type InputDataIndexType = InputDataType['indexes'][number];
export enum TabEnum {
chunk = 'chunk',
qa = 'qa',
image = 'image'
}
let indexClientId = 0;
const getIndexClientId = () => `dataset-index-${Date.now()}-${indexClientId++}`;
const clearEditingIndexDelay = 600;
const sortIndexesForDisplay = (indexes: InputDataIndexType[] = []) => {
const getOrder = (index: InputDataIndexType) => {
// Keep the editable custom indexes before the generated default index.
if (index.type === DatasetDataIndexTypeEnum.default) return 1;
return 0;
};
return [...indexes].sort((a, b) => getOrder(a) - getOrder(b));
};
const formatIndexesForForm = (
indexes: DatasetDataIndexItemType[] = [],
previousIndexes: InputDataIndexType[] = []
) => {
const previousIndexMap = previousIndexes.reduce<Record<string, InputDataIndexType>>(
(acc, index) => {
if (index.dataId) {
acc[index.dataId] = index;
}
return acc;
},
{}
);
return sortIndexesForDisplay(
indexes.map((item) => {
const previousIndex = item.dataId ? previousIndexMap[item.dataId] : undefined;
return {
...item,
clientId: previousIndex?.clientId || item.dataId || getIndexClientId(),
fold: previousIndex?.fold ?? true
};
})
);
};
const formatIndexesForRequest = (indexes: InputDataType['indexes'] = []) =>
indexes
.filter((item) => !!item.text?.trim())
.map((item) => ({
// Strip UI-only fields before submitting to the import API.
type: item.type,
text: item.text.trim()
}));
const formatDataForForm = (
data: Partial<Pick<InputDataType, 'q' | 'a' | 'imagePreivewUrl'>> & {
indexes?: DatasetDataIndexItemType[];
} = {},
dataId?: string,
previousIndexes?: InputDataIndexType[]
): InputDataType & { dataId?: string } => ({
...(dataId ? { dataId } : {}),
q: data.q || '',
a: data.a || '',
imagePreivewUrl: data.imagePreivewUrl,
indexes: formatIndexesForForm(data.indexes, previousIndexes)
});
const getInitialTab = ({
collectionType,
hasAnswer
}: {
collectionType: `${DatasetCollectionTypeEnum}`;
hasAnswer?: boolean;
}) => {
if (collectionType === DatasetCollectionTypeEnum.images) return TabEnum.image;
return hasAnswer ? TabEnum.qa : TabEnum.chunk;
};
export const useInputDataModal = ({
collectionId,
dataId,
defaultValue,
onSuccess
}: {
collectionId: string;
dataId?: string;
defaultValue?: { q?: string; a?: string; imagePreivewUrl?: string };
onSuccess: (data: InputDataType & { dataId: string }) => void;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { embeddingModelList, defaultModels } = useSystemStore();
const [currentTab, setCurrentTab] = useState<TabEnum>();
const [deletingIndexClientId, setDeletingIndexClientId] = useState<string>();
const [focusIndexClientId, setFocusIndexClientId] = useState<string>();
const [editingIndexClientId, setEditingIndexClientId] = useState<string>();
const { register, handleSubmit, reset, control, getValues, setValue } = useForm<InputDataType>({
shouldFocusError: false,
defaultValues: {
q: '',
a: '',
indexes: []
}
});
const {
fields: indexes,
prepend: prependIndexes,
remove: removeIndexes
} = useFieldArray({
control,
name: 'indexes'
});
const watchedIndexes = useWatch({
control,
name: 'indexes'
});
const imagePreivewUrl = useWatch({
control,
name: 'imagePreivewUrl'
});
// Saved snapshot keyed by clientId, so newly-created rows can keep their UI identity.
const savedIndexMapRef = useRef<Record<string, InputDataIndexType>>({});
// One index can only have one create/update/delete request in flight.
const pendingIndexClientIdsRef = useRef(new Set<string>());
const deletingIntentClientIdsRef = useRef(new Set<string>());
const queuedSaveIndexClientIdsRef = useRef(new Set<string>());
const saveIndexRunnerRef = useRef<(clientId: string) => Promise<unknown>>();
const deleteIndexRunnerRef = useRef<(clientId: string) => Promise<unknown>>();
const clearEditingIndexTimerRef = useRef<ReturnType<typeof setTimeout>>();
const resetSavedIndexes = useCallback((indexes: InputDataIndexType[] = []) => {
savedIndexMapRef.current = indexes.reduce<Record<string, InputDataIndexType>>((acc, index) => {
if (index.clientId) {
acc[index.clientId] = index;
}
return acc;
}, {});
}, []);
const getSuccessData = useCallback(
(indexes: InputDataType['indexes'] = getValues().indexes || []) => {
if (!dataId) return;
const formData = getValues();
return {
...formData,
indexes,
dataId
};
},
[dataId, getValues]
);
const findIndexByClientId = useCallback(
(clientId: string) => {
return (getValues().indexes || []).findIndex((item) => item.clientId === clientId);
},
[getValues]
);
const updateIndexFold = useCallback(
(clientId: string, fold: boolean) => {
const currentIndex = findIndexByClientId(clientId);
if (currentIndex < 0) return;
const targetIndex = getValues().indexes?.[currentIndex];
if (!targetIndex) return;
setValue(`indexes.${currentIndex}`, {
...targetIndex,
fold
});
},
[findIndexByClientId, getValues, setValue]
);
const prependCustomIndex = useCallback(() => {
const clientId = getIndexClientId();
setFocusIndexClientId(clientId);
prependIndexes({
type: DatasetDataIndexTypeEnum.custom,
text: '',
clientId,
fold: false
});
}, [prependIndexes]);
const clearFocusIndexClientId = useCallback((clientId: string) => {
setFocusIndexClientId((state) => (state === clientId ? undefined : state));
}, []);
const markEditingIndex = useCallback((clientId: string) => {
if (clearEditingIndexTimerRef.current) {
clearTimeout(clearEditingIndexTimerRef.current);
}
setEditingIndexClientId(clientId);
}, []);
const clearEditingIndex = useCallback((clientId: string) => {
if (clearEditingIndexTimerRef.current) {
clearTimeout(clearEditingIndexTimerRef.current);
}
clearEditingIndexTimerRef.current = setTimeout(() => {
setEditingIndexClientId((state) => (state === clientId ? undefined : state));
}, clearEditingIndexDelay);
}, []);
useEffect(() => {
return () => {
if (clearEditingIndexTimerRef.current) {
clearTimeout(clearEditingIndexTimerRef.current);
}
};
}, []);
const refreshDataForm = useCallback(
async (targetDataId: string) => {
const currentIndexes = getValues().indexes || [];
const latestData = await getDatasetDataItemById(targetDataId);
const refreshedData = formatDataForForm(
latestData,
targetDataId,
currentIndexes
) as InputDataType & {
dataId: string;
};
resetSavedIndexes(refreshedData.indexes);
reset(refreshedData);
return refreshedData;
},
[getValues, reset, resetSavedIndexes]
);
const { data: collection = defaultCollectionDetail, loading: initLoading } = useRequest(
async () => {
const [collection, dataItem] = await Promise.all([
getDatasetCollectionById(collectionId),
...(dataId ? [getDatasetDataItemById(dataId)] : [])
]);
// Opening another data item should not inherit optimistic state from the previous one.
pendingIndexClientIdsRef.current.clear();
deletingIntentClientIdsRef.current.clear();
queuedSaveIndexClientIdsRef.current.clear();
setDeletingIndexClientId(undefined);
setFocusIndexClientId(undefined);
setEditingIndexClientId(undefined);
const initialData = dataItem || defaultValue;
setCurrentTab(
getInitialTab({
collectionType: collection.type,
hasAnswer: !!initialData?.a
})
);
const formData = formatDataForForm(initialData);
resetSavedIndexes(formData.indexes);
reset(formData);
return collection;
},
{
manual: false,
refreshDeps: [collectionId, dataId, defaultValue]
}
);
const { runAsync: sureImportData, loading: isImporting } = useRequest(
async (e: InputDataType) => {
const data = { ...e };
const postData: Parameters<typeof postInsertData2Dataset>[0] = {
collectionId: collection._id,
q: e.q,
a: currentTab === TabEnum.qa ? e.a : '',
indexes: formatIndexesForRequest(e.indexes)
};
const dataId = await postInsertData2Dataset(postData);
return {
...data,
dataId
};
},
{
refreshDeps: [currentTab],
successToast: t('common:dataset.data.Input Success Tip'),
onSuccess(e) {
reset({
...e,
q: '',
a: '',
indexes: []
});
resetSavedIndexes();
onSuccess(e);
},
errorToast: t('dataset:common.error.unKnow')
}
);
const { runAsync: onUpdateData, loading: isUpdating } = useRequest(
async (e: InputDataType) => {
if (!dataId) return Promise.reject(t('common:error.unKnow'));
await putDatasetDataById({
dataId,
q: e.q,
a: currentTab === TabEnum.qa ? e.a : ''
});
return refreshDataForm(dataId);
},
{
refreshDeps: [currentTab, refreshDataForm],
successToast: t('common:dataset.data.Update Success Tip'),
onSuccess(data) {
onSuccess(data);
}
}
);
const { runAsync: onSaveIndex } = useRequest(
async (clientId: string) => {
if (deletingIntentClientIdsRef.current.has(clientId)) return;
if (pendingIndexClientIdsRef.current.has(clientId)) {
queuedSaveIndexClientIdsRef.current.add(clientId);
return;
}
const currentIndex = findIndexByClientId(clientId);
if (currentIndex < 0) return;
const targetIndex = getValues().indexes[currentIndex];
const text = targetIndex?.text?.trim() || '';
const type = targetIndex?.type || DatasetDataIndexTypeEnum.custom;
if (type === DatasetDataIndexTypeEnum.default) {
return;
}
if (!text) {
if (!targetIndex?.dataId) {
// Empty unsaved rows are local drafts, so dropping them should stay silent.
removeIndexes(currentIndex);
}
return;
}
if (
targetIndex?.dataId &&
text === savedIndexMapRef.current[clientId]?.text?.trim() &&
type === (savedIndexMapRef.current[clientId]?.type || DatasetDataIndexTypeEnum.custom)
) {
return;
}
if (!dataId) return;
pendingIndexClientIdsRef.current.add(clientId);
let shouldSaveLatest = false;
let saveError: unknown;
try {
const indexDataId = targetIndex.dataId;
// dataId means this row exists in vector storage; otherwise create it first.
const { index: savedIndex } = indexDataId
? await updateDatasetDataIndex({
dataId,
indexDataId,
type,
text
})
: await createDatasetDataIndex({
dataId,
type,
text
});
const latestIndex = findIndexByClientId(clientId);
if (latestIndex < 0) return;
const latestTargetIndex = getValues().indexes[latestIndex];
if (!latestTargetIndex) return;
const savedFormIndex = {
...savedIndex,
clientId,
fold: latestTargetIndex.fold
};
savedIndexMapRef.current[clientId] = savedFormIndex;
// If the user keeps typing while save is pending, keep the latest text in the form
// but attach the backend id returned by the create request.
const hasChangedDuringSave =
latestTargetIndex.text?.trim() !== text ||
(latestTargetIndex.type || DatasetDataIndexTypeEnum.custom) !== type;
shouldSaveLatest =
hasChangedDuringSave || queuedSaveIndexClientIdsRef.current.has(clientId);
queuedSaveIndexClientIdsRef.current.delete(clientId);
const nextFormIndex = hasChangedDuringSave
? {
...latestTargetIndex,
dataId: savedIndex.dataId
}
: savedFormIndex;
const currentIndexes = getValues().indexes || [];
const nextIndexes = currentIndexes.map((item, i) =>
i === latestIndex ? nextFormIndex : item
);
setValue(`indexes.${latestIndex}`, nextFormIndex);
const successData = getSuccessData(nextIndexes);
if (successData) {
onSuccess(successData);
}
} catch (error) {
saveError = error;
} finally {
pendingIndexClientIdsRef.current.delete(clientId);
}
if (deletingIntentClientIdsRef.current.has(clientId)) {
await deleteIndexRunnerRef.current?.(clientId);
return;
}
if (saveError) {
return Promise.reject(saveError);
}
if (shouldSaveLatest) {
await saveIndexRunnerRef.current?.(clientId);
}
},
{
refreshDeps: [dataId, findIndexByClientId, getSuccessData, getValues, removeIndexes, setValue]
}
);
useEffect(() => {
saveIndexRunnerRef.current = onSaveIndex;
}, [onSaveIndex]);
const markDeletingIndex = useCallback((clientId: string) => {
deletingIntentClientIdsRef.current.add(clientId);
}, []);
const { runAsync: onDeleteIndex, loading: isDeletingIndex } = useRequest(
async (clientId: string) => {
const currentIndex = findIndexByClientId(clientId);
if (currentIndex < 0) {
deletingIntentClientIdsRef.current.delete(clientId);
setEditingIndexClientId((state) => (state === clientId ? undefined : state));
return;
}
if (!dataId) {
removeIndexes(currentIndex);
deletingIntentClientIdsRef.current.delete(clientId);
setEditingIndexClientId((state) => (state === clientId ? undefined : state));
return;
}
const targetIndex = getValues().indexes[currentIndex];
const indexDataId = targetIndex?.dataId;
if (!indexDataId) {
// Draft rows are not persisted, so deleting them is only a field-array operation.
removeIndexes(currentIndex);
deletingIntentClientIdsRef.current.delete(clientId);
setEditingIndexClientId((state) => (state === clientId ? undefined : state));
return;
}
if (pendingIndexClientIdsRef.current.has(clientId)) return;
pendingIndexClientIdsRef.current.add(clientId);
setDeletingIndexClientId(clientId);
try {
await deleteDatasetDataIndex({
dataId,
indexDataId
});
toast({
title: t('common:delete_success'),
status: 'success'
});
delete savedIndexMapRef.current[clientId];
const latestIndex = findIndexByClientId(clientId);
if (latestIndex < 0) return;
const nextIndexes = (getValues().indexes || []).filter(
(item) => item.clientId !== clientId
);
removeIndexes(latestIndex);
const successData = getSuccessData(nextIndexes);
if (successData) {
onSuccess(successData);
}
} finally {
setDeletingIndexClientId((state) => (state === clientId ? undefined : state));
setEditingIndexClientId((state) => (state === clientId ? undefined : state));
pendingIndexClientIdsRef.current.delete(clientId);
deletingIntentClientIdsRef.current.delete(clientId);
}
},
{
refreshDeps: [dataId, findIndexByClientId, getSuccessData, getValues, removeIndexes, t, toast]
}
);
useEffect(() => {
deleteIndexRunnerRef.current = onDeleteIndex;
}, [onDeleteIndex]);
const maxToken = useMemo(() => {
const vectorModel =
embeddingModelList.find((item) => item.model === collection.dataset.vectorModel) ||
defaultModels.embedding;
return vectorModel?.maxToken || 2000;
}, [collection.dataset.vectorModel, defaultModels.embedding, embeddingModelList]);
const submitData = handleSubmit((data) => (dataId ? onUpdateData(data) : sureImportData(data)));
const showTabs = currentTab === TabEnum.chunk || currentTab === TabEnum.qa;
return {
collection,
currentTab,
deletingIndexClientId,
editingIndexClientId,
focusIndexClientId,
imagePreivewUrl,
indexes,
initLoading,
isDeletingIndex,
isImporting,
isUpdating,
maxToken,
register,
showTabs,
submitData,
watchedIndexes,
onDeleteIndex,
onSaveIndex,
prependCustomIndex,
setCurrentTab,
clearFocusIndexClientId,
markEditingIndex,
clearEditingIndex,
markDeletingIndex,
updateIndexFold
};
};
projects/app/src/pageComponents/dataset/list/CreateModal.tsx
import React, { useMemo } from 'react';
import { Box, Flex, Button, ModalFooter, ModalBody, Input, HStack } from '@chakra-ui/react';
import { Box, Flex, Button, Input, HStack } from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
import { useRouter } from 'next/router';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import Avatar from '@fastgpt/web/components/common/Avatar';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyModal from '@fastgpt/web/components/v2/common/MyModal';
import { postCreateDataset } from '@/web/core/dataset/api';
import type { CreateDatasetBody } from '@fastgpt/global/openapi/core/dataset/api';
import { useTranslation } from 'next-i18next';
import { DatasetTypeEnum, DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
import AIModelSelector from '@/components/Select/AIModelSelector';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import ComplianceTip from '@/components/common/ComplianceTip/index';
import MyIcon from '@fastgpt/web/components/common/Icon';
......@@ -42,7 +41,6 @@ const CreateModal = ({
const { t } = useTranslation();
const router = useRouter();
const { defaultModels, embeddingModelList, llmModelList, getVlmModelList } = useSystemStore();
const { isPc } = useSystem();
const filterNotHiddenVectorModelList = embeddingModelList.filter((item) => !item.hidden);
......@@ -66,6 +64,11 @@ const CreateModal = ({
const vectorModel = watch('vectorModel');
const agentModel = watch('agentModel');
const vlmModel = watch('vlmModel');
const showApiDatasetForm =
type === DatasetTypeEnum.apiDataset ||
type === DatasetTypeEnum.feishu ||
type === DatasetTypeEnum.yuque ||
type === DatasetTypeEnum.dingtalk;
const { Component: AvatarUploader, handleFileSelectorOpen: handleAvatarSelectorOpen } =
useUploadAvatar(getUploadAvatarPresignedUrl, {
......@@ -88,181 +91,197 @@ const CreateModal = ({
return (
<MyModal
title={
<Flex alignItems={'center'} ml={-3}>
<Avatar
w={'20px'}
h={'20px'}
borderRadius={'xs'}
src={DatasetTypeMap[type].avatar}
pr={'10px'}
/>
{t('common:core.dataset.Create dataset', { name: t(DatasetTypeMap[type].label) })}
</Flex>
}
isOpen
onClose={onClose}
isCentered={!isPc}
w={'490px'}
size={'md'}
iconSrc={DatasetTypeMap[type].avatar}
title={t('dataset:create_dataset_title', { name: t(DatasetTypeMap[type].label) })}
contentPx={'32px'}
contentPy={'32px'}
borderRadius={'10px'}
>
<ModalBody py={6} px={9}>
<Box>
<Flex justify={'space-between'}>
<Box color={'myGray.900'} fontWeight={500} fontSize={'sm'}>
{t('common:input_name')}
<Flex
flexDirection={'column'}
alignItems={'flex-start'}
minH={showApiDatasetForm ? undefined : '338px'}
>
<Flex w={'100%'} flexDirection={'column'} gap={4}>
<Box w={'100%'}>
<Flex justify={'space-between'}>
<Box color={'myGray.900'} fontWeight={500} fontSize={'sm'}>
{t('common:Name')}
</Box>
{DatasetTypeMap[type]?.courseUrl && (
<Flex
as={'span'}
alignItems={'center'}
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() => window.open(getDocPath(DatasetTypeMap[type].courseUrl!), '_blank')}
>
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}
</Flex>
)}
</Flex>
<Flex mt={'12px'} alignItems={'center'}>
<MyTooltip label={t('common:click_select_avatar')}>
<Avatar
flexShrink={0}
src={avatar}
w={['28px', '32px']}
h={['28px', '32px']}
cursor={'pointer'}
borderRadius={'md'}
onClick={handleAvatarSelectorOpen}
/>
</MyTooltip>
<Input
ml={4}
flex={1}
autoFocus
bg={'myWhite.600'}
fontSize={'14px'}
placeholder={t('dataset:dataset_name_placeholder')}
maxLength={30}
{...register('name', {
required: true
})}
/>
</Flex>
</Box>
<Flex
w={'100%'}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
>
<HStack
spacing={1}
alignItems={'center'}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
>
<Box>{t('common:core.ai.model.Vector Model')}</Box>
<QuestionTip label={t('common:core.dataset.embedding model tip')} />
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={vectorModel}
list={filterNotHiddenVectorModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onChange={(e) => {
setValue('vectorModel' as const, e);
}}
/>
</Box>
{DatasetTypeMap[type]?.courseUrl && (
<Flex
as={'span'}
alignItems={'center'}
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() => window.open(getDocPath(DatasetTypeMap[type].courseUrl!), '_blank')}
>
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}
</Flex>
)}
</Flex>
<Flex mt={'12px'} alignItems={'center'}>
<MyTooltip label={t('common:click_select_avatar')}>
<Avatar
flexShrink={0}
src={avatar}
w={['28px', '32px']}
h={['28px', '32px']}
cursor={'pointer'}
borderRadius={'md'}
onClick={handleAvatarSelectorOpen}
<Flex
w={'100%'}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
>
<HStack
spacing={1}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
>
<Box>{t('common:core.ai.model.Dataset Agent Model')}</Box>
<QuestionTip label={t('dataset:file_model_function_tip')} />
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={agentModel}
list={llmModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onChange={(e) => {
setValue('agentModel', e);
}}
/>
</MyTooltip>
<Input
ml={3}
flex={1}
autoFocus
bg={'myWhite.600'}
placeholder={t('common:Name')}
maxLength={30}
{...register('name', {
required: true
})}
/>
</Box>
</Flex>
</Box>
<Flex
mt={6}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
>
<HStack
spacing={1}
alignItems={'center'}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
<Flex
w={'100%'}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
>
<Box>{t('common:core.ai.model.Vector Model')}</Box>
<QuestionTip label={t('common:core.dataset.embedding model tip')} />
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={vectorModel}
list={filterNotHiddenVectorModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onChange={(e) => {
setValue('vectorModel' as const, e);
}}
/>
</Box>
<HStack
spacing={1}
alignItems={'center'}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
>
<Box>{t('dataset:vllm_model')}</Box>
<QuestionTip label={t('dataset:vllm_model_tip')} />
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={vlmModel}
list={vllmModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onChange={(e) => {
setValue('vlmModel', e);
}}
/>
</Box>
</Flex>
</Flex>
<Flex
mt={6}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
>
<HStack
spacing={1}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
{showApiDatasetForm && (
<Box
mt={4}
w={'100%'}
sx={{
'& > *:first-of-type': {
mt: '0 !important'
}
}}
>
<Box>{t('common:core.ai.model.Dataset Agent Model')}</Box>
<QuestionTip label={t('dataset:file_model_function_tip')} />
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={agentModel}
list={llmModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onChange={(e) => {
setValue('agentModel', e);
}}
/>
<ApiDatasetForm type={type} form={form} controlWidth={['100%', '300px']} />
</Box>
</Flex>
)}
<Flex
mt={6}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
>
<HStack
spacing={1}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
<Flex mt={6} w={'100%'} justifyContent={'flex-end'} gap={3}>
<Button variant={'whiteBase'} fontSize={'12px'} onClick={onClose}>
{t('common:Close')}
</Button>
<Button
fontSize={'12px'}
isLoading={creating}
onClick={handleSubmit((data) => onclickCreate(data))}
>
<Box>{t('dataset:vllm_model')}</Box>
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={vlmModel}
list={vllmModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onChange={(e) => {
setValue('vlmModel', e);
}}
/>
</Box>
{t('common:Create')}
</Button>
</Flex>
{/* @ts-ignore */}
<ApiDatasetForm type={type} form={form} />
</ModalBody>
<ModalFooter px={9}>
<Button variant={'whiteBase'} mr={3} onClick={onClose}>
{t('common:Close')}
</Button>
<Button isLoading={creating} onClick={handleSubmit((data) => onclickCreate(data))}>
{t('common:comfirn_create')}
</Button>
</ModalFooter>
<ComplianceTip pb={6} pt={0} px={9} type={'dataset'} />
<ComplianceTip pb={0} pt={0} px={0} type={'dataset'} />
<AvatarUploader />
<AvatarUploader />
</Flex>
</MyModal>
);
};
......
projects/app/src/pages/api/core/dataset/data/delete.ts
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
import { deleteDatasetData } from '@/service/core/dataset/data/controller';
import { deleteDatasetData } from '@/service/core/dataset/data/data';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { addAuditLog } from '@fastgpt/service/support/user/audit/util';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { DeleteDatasetDataQuerySchema } from '@fastgpt/global/openapi/core/dataset/data/api';
import {
DeleteDatasetDataQuerySchema,
DeleteDatasetDataResponseSchema,
type DeleteDatasetDataResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
async function handler(req: ApiRequestProps) {
async function handler(req: ApiRequestProps): Promise<DeleteDatasetDataResponse> {
const { id: dataId } = DeleteDatasetDataQuerySchema.parse(req.query);
// 凭证校验
......@@ -34,7 +38,7 @@ async function handler(req: ApiRequestProps) {
}
});
})();
return 'success';
return DeleteDatasetDataResponseSchema.parse('success');
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/data/index/create.ts 0 → 100644
import { createDatasetDataIndex } from '@/service/core/dataset/data/dataIndex';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { NextAPI } from '@/service/middleware/entry';
import {
CreateDatasetDataIndexBodySchema,
DatasetDataIndexResponseSchema,
type DatasetDataIndexResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { addAuditLog, getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
async function handler(req: ApiRequestProps): Promise<DatasetDataIndexResponse> {
const { dataId, type, text } = CreateDatasetDataIndexBodySchema.parse(req.body);
const { datasetData, tmbId, teamId, collection } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: WritePermissionVal
});
const { index, tokens } = await createDatasetDataIndex({
data: datasetData,
type,
text,
model: collection.dataset.vectorModel
});
if (tokens > 0) {
pushGenerateVectorUsage({
teamId,
tmbId,
inputTokens: tokens,
model: collection.dataset.vectorModel
});
}
addAuditLog({
tmbId,
teamId,
event: AuditEventEnum.UPDATE_DATA,
params: {
collectionName: collection.name,
datasetName: collection.dataset?.name || '',
datasetType: getI18nDatasetType(collection.dataset?.type || '')
}
});
return DatasetDataIndexResponseSchema.parse({
index
});
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/data/index/delete.ts 0 → 100644
import { deleteDatasetDataIndex } from '@/service/core/dataset/data/dataIndex';
import { NextAPI } from '@/service/middleware/entry';
import {
DeleteDatasetDataIndexBodySchema,
DeleteDatasetDataIndexResponseSchema,
type DeleteDatasetDataIndexResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { addAuditLog, getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
async function handler(req: ApiRequestProps): Promise<DeleteDatasetDataIndexResponse> {
const { dataId, indexDataId } = DeleteDatasetDataIndexBodySchema.parse(req.body);
const { datasetData, tmbId, teamId, collection } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: WritePermissionVal
});
await deleteDatasetDataIndex({
data: datasetData,
indexDataId
});
addAuditLog({
tmbId,
teamId,
event: AuditEventEnum.UPDATE_DATA,
params: {
collectionName: collection.name,
datasetName: collection.dataset?.name || '',
datasetType: getI18nDatasetType(collection.dataset?.type || '')
}
});
return DeleteDatasetDataIndexResponseSchema.parse({});
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/data/index/update.ts 0 → 100644
import { updateDatasetDataIndex } from '@/service/core/dataset/data/dataIndex';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { NextAPI } from '@/service/middleware/entry';
import {
UpdateDatasetDataIndexBodySchema,
DatasetDataIndexResponseSchema,
type DatasetDataIndexResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { addAuditLog, getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
async function handler(req: ApiRequestProps): Promise<DatasetDataIndexResponse> {
const { dataId, indexDataId, type, text } = UpdateDatasetDataIndexBodySchema.parse(req.body);
const { datasetData, tmbId, teamId, collection } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: WritePermissionVal
});
const { index, tokens } = await updateDatasetDataIndex({
data: datasetData,
indexDataId,
type,
text,
model: collection.dataset.vectorModel
});
if (tokens > 0) {
pushGenerateVectorUsage({
teamId,
tmbId,
inputTokens: tokens,
model: collection.dataset.vectorModel
});
}
addAuditLog({
tmbId,
teamId,
event: AuditEventEnum.UPDATE_DATA,
params: {
collectionName: collection.name,
datasetName: collection.dataset?.name || '',
datasetType: getI18nDatasetType(collection.dataset?.type || '')
}
});
return DatasetDataIndexResponseSchema.parse({
index
});
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/data/insertData.ts
......@@ -4,7 +4,7 @@
*/
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
import { hasSameValue } from '@/service/core/dataset/data/utils';
import { insertData2Dataset } from '@/service/core/dataset/data/controller';
import { createDatasetData } from '@/service/core/dataset/data/data';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
......@@ -41,7 +41,7 @@ async function handler(req: ApiRequestProps): Promise<InsertDataResponse> {
const [
{
dataset: { _id: datasetId, vectorModel, agentModel },
dataset: { _id: datasetId, vectorModel },
indexPrefixTitle,
name
}
......@@ -64,7 +64,7 @@ async function handler(req: ApiRequestProps): Promise<InsertDataResponse> {
a: formatA
});
const { insertId, tokens } = await insertData2Dataset({
const { insertId, tokens } = await createDatasetData({
teamId,
tmbId,
datasetId,
......
projects/app/src/pages/api/core/dataset/data/update.ts
import { updateData2Dataset } from '@/service/core/dataset/data/controller';
import {
updateDatasetDataByIndexes,
updateDatasetDataDefaultIndexes
} from '@/service/core/dataset/data/data';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
......@@ -7,10 +10,15 @@ import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { addAuditLog } from '@fastgpt/service/support/user/audit/util';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { UpdateDatasetDataBodySchema } from '@fastgpt/global/openapi/core/dataset/data/api';
import {
UpdateDatasetDataBodySchema,
UpdateDatasetDataResponseSchema,
type UpdateDatasetDataResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
async function handler(req: ApiRequestProps) {
const { dataId, q, a, indexes = [] } = UpdateDatasetDataBodySchema.parse(req.body);
async function handler(req: ApiRequestProps): Promise<UpdateDatasetDataResponse> {
const { dataId, q, a, indexes } = UpdateDatasetDataBodySchema.parse(req.body);
const hasIndexes = Object.hasOwn(req.body, 'indexes');
// auth data permission
const {
......@@ -21,7 +29,8 @@ async function handler(req: ApiRequestProps) {
},
teamId,
tmbId,
collection
collection,
datasetData
} = await authDatasetData({
req,
authToken: true,
......@@ -30,36 +39,61 @@ async function handler(req: ApiRequestProps) {
per: WritePermissionVal
});
if (q || a || indexes.length > 0) {
const { tokens } = await updateData2Dataset({
if (hasIndexes) {
const { tokens } = await updateDatasetDataByIndexes({
dataId,
q,
a,
indexes,
indexes: indexes ?? [],
model: vectorModel,
indexPrefix: indexPrefixTitle ? `# ${name}` : undefined
});
pushGenerateVectorUsage({
teamId,
tmbId,
inputTokens: tokens,
model: vectorModel
if (tokens > 0) {
pushGenerateVectorUsage({
teamId,
tmbId,
inputTokens: tokens,
model: vectorModel
});
}
} else {
const nextQ = q || datasetData.q || '';
const nextA = a ?? datasetData.a ?? '';
const { tokens } = await updateDatasetDataDefaultIndexes({
dataId,
q: nextQ,
a: nextA,
model: vectorModel,
indexSize: collection.indexSize,
indexPrefix: indexPrefixTitle ? `# ${name}` : undefined
});
(() => {
addAuditLog({
tmbId,
if (tokens > 0) {
pushGenerateVectorUsage({
teamId,
event: AuditEventEnum.UPDATE_DATA,
params: {
collectionName: collection.name,
datasetName: collection.dataset?.name || '',
datasetType: getI18nDatasetType(collection.dataset?.type || '')
}
tmbId,
inputTokens: tokens,
model: vectorModel
});
})();
}
}
(() => {
addAuditLog({
tmbId,
teamId,
event: AuditEventEnum.UPDATE_DATA,
params: {
collectionName: collection.name,
datasetName: collection.dataset?.name || '',
datasetType: getI18nDatasetType(collection.dataset?.type || '')
}
});
})();
return UpdateDatasetDataResponseSchema.parse({});
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/file/getSearchTestImagePreviewUrls.ts 0 → 100644
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { addHours } from 'date-fns';
import { S3Buckets } from '@fastgpt/service/common/s3/config/constants';
import { isS3ObjectKey, jwtSignS3DownloadToken } from '@fastgpt/service/common/s3/utils';
import {
GetSearchTestImagePreviewUrlsBodySchema,
GetSearchTestImagePreviewUrlsResponseSchema,
type GetSearchTestImagePreviewUrlsBody,
type GetSearchTestImagePreviewUrlsResponse
} from '@fastgpt/global/openapi/core/dataset/file/api';
async function handler(
req: ApiRequestProps<GetSearchTestImagePreviewUrlsBody>
): Promise<GetSearchTestImagePreviewUrlsResponse> {
const { datasetId, keys } = GetSearchTestImagePreviewUrlsBodySchema.parse(req.body);
const { teamId } = await authDataset({
datasetId,
per: ReadPermissionVal,
req,
authToken: true,
authApiKey: true
});
const result = keys
.filter((key) => isS3ObjectKey(key, 'temp') && key.startsWith(`temp/${teamId}/`))
.map((key) => ({
key,
previewUrl: jwtSignS3DownloadToken({
objectKey: key,
bucketName: S3Buckets.private,
expiredTime: addHours(new Date(), 1)
})
}));
return GetSearchTestImagePreviewUrlsResponseSchema.parse(result);
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/file/presignSearchTestImage.ts 0 → 100644
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { authFrequencyLimit } from '@fastgpt/service/common/system/frequencyLimit/utils';
import { addSeconds } from 'date-fns';
import { getTeamPlanStatus } from '@fastgpt/service/support/wallet/sub/utils';
import { imageFileType } from '@fastgpt/global/common/file/constants';
import { parseAllowedExtensions } from '@fastgpt/service/common/s3/utils/uploadConstraints';
import { getFileS3Key } from '@fastgpt/service/common/s3/utils';
import { S3PrivateBucket } from '@fastgpt/service/common/s3/buckets/private';
import {
PresignSearchTestImageBodySchema,
PresignSearchTestImageResponseSchema,
type PresignSearchTestImageBody,
type PresignSearchTestImageResponse
} from '@fastgpt/global/openapi/core/dataset/file/api';
async function handler(
req: ApiRequestProps<PresignSearchTestImageBody>
): Promise<PresignSearchTestImageResponse> {
const { datasetId, filename } = PresignSearchTestImageBodySchema.parse(req.body);
const { teamId, userId } = await authDataset({
datasetId,
per: ReadPermissionVal,
req,
authToken: true,
authApiKey: true
});
const planStatus = await getTeamPlanStatus({ teamId });
await authFrequencyLimit({
eventId: `${userId}-uploadfile`,
maxAmount: planStatus.standard?.maxUploadFileCount || global.feConfigs.uploadFileMaxAmount,
expiredTime: addSeconds(new Date(), 30),
num: 1
});
const bucket = new S3PrivateBucket();
const { fileKey } = getFileS3Key.temp({ teamId, filename });
const result = await bucket.createPresignedPutUrl(
{ rawKey: fileKey, filename },
{
expiredHours: 3,
maxFileSize: planStatus.standard?.maxUploadFileSize ?? global.feConfigs.uploadFileMaxSize,
uploadConstraints: {
allowedExtensions: parseAllowedExtensions(imageFileType)
}
}
);
return PresignSearchTestImageResponseSchema.parse(result);
}
export default NextAPI(handler);
projects/app/src/service/core/dataset/data/controller.ts deleted 100644 → 0
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { insertDatasetDataVector } from '@fastgpt/service/common/vectorDB/controller';
import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index';
import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorDB/controller';
import { pushCollectionUpdateJob } from '@fastgpt/service/core/dataset/collection/mq';
import type {
UpdateDatasetDataPropsType,
DatasetDataIndexItemType,
DatasetDataItemType,
CreateDatasetDataPropsType
} from '@fastgpt/global/core/dataset/type';
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { type ClientSession } from '@fastgpt/service/common/mongo';
import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken';
import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils';
import { text2Chunks } from '@fastgpt/service/worker/function';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import { removeS3TTL } from '@fastgpt/service/common/s3/utils';
const formatIndexes = async ({
indexes = [],
q,
a = '',
indexSize,
maxIndexSize,
indexPrefix
}: {
indexes?: (Omit<DatasetDataIndexItemType, 'dataId'> & { dataId?: string })[];
q: string;
a?: string;
indexSize: number;
maxIndexSize: number;
indexPrefix?: string;
}): Promise<
{
type: DatasetDataIndexTypeEnum;
text: string;
dataId?: string;
}[]
> => {
const formatText = (text: string) => {
if (indexPrefix && !text.startsWith(indexPrefix)) {
return `${indexPrefix}\n${text}`;
}
return text;
};
/* get dataset data default index */
const getDefaultIndex = async ({
q = '',
a,
indexSize
}: {
q?: string;
a?: string;
indexSize: number;
}) => {
const qChunks = (
await text2Chunks({
text: q,
chunkSize: indexSize,
maxSize: maxIndexSize
})
).chunks;
const aChunks = a
? (await text2Chunks({ text: a, chunkSize: indexSize, maxSize: maxIndexSize })).chunks
: [];
return [
...qChunks.map((text) => ({
text: formatText(text),
type: DatasetDataIndexTypeEnum.default
})),
...aChunks.map((text) => ({
text: formatText(text),
type: DatasetDataIndexTypeEnum.default
}))
];
};
// If index not type, set it to custom
indexes = indexes.map((item) => ({
text: typeof item.text === 'string' ? item.text : String(item.text),
type: item.type || DatasetDataIndexTypeEnum.custom,
dataId: item.dataId
}));
// Recompute default indexes, Merge ids of the same index, reduce the number of rebuilds
const defaultIndexes = await getDefaultIndex({ q, a, indexSize });
const concatDefaultIndexes = defaultIndexes.map((item) => {
const oldIndex = indexes!.find((index) => index.text === item.text);
if (oldIndex) {
return {
type: DatasetDataIndexTypeEnum.default,
text: item.text,
dataId: oldIndex.dataId
};
} else {
return item;
}
});
// 其他索引不能与默认索引相同，且不能自己有重复
indexes = indexes.filter(
(item, index, self) =>
item.type !== DatasetDataIndexTypeEnum.default &&
!concatDefaultIndexes.find((t) => t.text === item.text) &&
index === self.findIndex((t) => t.text === item.text)
);
indexes.push(...concatDefaultIndexes);
const chekcIndexes = (
await Promise.all(
indexes.map(async (item) => {
if (item.type === DatasetDataIndexTypeEnum.default) {
return item;
}
// If oversize tokens, split it
const tokens = await countPromptTokens(item.text);
if (tokens > maxIndexSize) {
const splitText = (
await text2Chunks({
text: item.text,
chunkSize: indexSize,
maxSize: maxIndexSize
})
).chunks;
return splitText.map((text) => ({
text,
type: item.type
}));
}
return item;
})
)
)
.flat()
.filter((item) => !!item.text.trim());
// Add prefix
const prefixIndexes = indexPrefix
? chekcIndexes.map((index) => {
if (index.type === DatasetDataIndexTypeEnum.custom) return index;
return {
...index,
text: formatText(index.text)
};
})
: chekcIndexes;
return prefixIndexes;
};
/* insert data.
* 1. create data id
* 2. insert pg
* 3. create mongo data
*/
export async function insertData2Dataset({
teamId,
tmbId,
datasetId,
collectionId,
q,
a,
imageId,
chunkIndex = 0,
indexSize = 512,
indexes,
indexPrefix,
embeddingModel,
imageDescMap,
session
}: CreateDatasetDataPropsType & {
embeddingModel: string;
indexSize?: number;
imageDescMap?: Record<string, string>;
session?: ClientSession;
}) {
if (!q || !datasetId || !collectionId || !embeddingModel) {
return Promise.reject('q, datasetId, collectionId, embeddingModel is required');
}
if (String(teamId) === String(tmbId)) {
return Promise.reject("teamId and tmbId can't be the same");
}
const embModel = getEmbeddingModel(embeddingModel);
indexSize = Math.min(embModel.maxToken, indexSize);
// 1. Get vector indexes and insert
// Empty indexes check, if empty, create default index
const newIndexes = await formatIndexes({
indexes,
q,
a,
indexSize,
maxIndexSize: embModel.maxToken,
indexPrefix
});
// insert to vector store
const { tokens, insertIds } = await insertDatasetDataVector({
inputs: newIndexes.map((item) => item.text),
model: embModel,
teamId,
datasetId,
collectionId
});
const results = newIndexes.map((item, index) => ({
...item,
dataId: insertIds[index]
}));
const [{ _id }] = await MongoDatasetData.create(
[
{
teamId,
tmbId,
datasetId,
collectionId,
q,
a,
imageId,
imageDescMap,
chunkIndex,
indexes: results
}
],
{ session, ordered: true }
);
// 3. Create mongo data text
await MongoDatasetDataText.create(
[
{
teamId,
datasetId,
collectionId,
dataId: _id,
fullTextToken: await jiebaSplit({ text: `${q}\n${a}`.trim() })
}
],
{ session, ordered: true }
);
// 只移除图片数据集的图片的 TTL
if (isS3ObjectKey(imageId, 'dataset')) {
await removeS3TTL({ key: imageId, bucketName: 'private', session });
}
// Trigger collection update (async, with 5s delay and debounce)
pushCollectionUpdateJob({
collectionId: String(collectionId),
datasetId: String(datasetId),
teamId: String(teamId)
});
return {
insertId: _id,
tokens
};
}
/**
* Update data(indexes overwrite)
* 1. compare indexes
* 2. insert new pg data
* session run:
* 3. update mongo data(session run)
* 4. delete old pg data
*/
type PatchIndexesProps =
| {
type: 'create';
index: Omit<DatasetDataIndexItemType, 'dataId'> & {
dataId?: string;
};
}
| {
type: 'update';
index: DatasetDataIndexItemType;
}
| {
type: 'delete';
index: DatasetDataIndexItemType;
}
| {
type: 'unChange';
index: DatasetDataIndexItemType;
};
export async function updateData2Dataset({
dataId,
q = '',
a,
indexes,
model,
indexSize = 512,
indexPrefix
}: UpdateDatasetDataPropsType & { model: string; indexSize?: number }) {
if (!Array.isArray(indexes)) {
return Promise.reject('indexes is required');
}
// 1. Get mongo data
const mongoData = await MongoDatasetData.findById(dataId);
if (!mongoData) return Promise.reject('core.dataset.error.Data not found');
// 2. Compute indexes
const formatIndexesResult = await formatIndexes({
indexes,
q,
a,
indexSize,
maxIndexSize: getEmbeddingModel(model).maxToken,
indexPrefix
});
// 3. Patch indexes, create, update, delete
const patchResult: PatchIndexesProps[] = [];
// find database indexes in new Indexes, if have not, delete it
for (const item of mongoData.indexes) {
const index = formatIndexesResult.find((index) => index.dataId === item.dataId);
if (!index) {
patchResult.push({
type: 'delete',
index: item
});
}
}
for (const item of formatIndexesResult) {
if (!item.dataId) {
patchResult.push({
type: 'create',
index: item
});
} else {
const index = mongoData.indexes.find((index) => index.dataId === item.dataId);
if (!index) continue;
// Not change
if (index.text === item.text) {
patchResult.push({
type: 'unChange',
index: {
...item,
dataId: index.dataId
}
});
} else {
// index Update
patchResult.push({
type: 'update',
index: {
...item,
dataId: index.dataId
}
});
}
}
}
const deleteVectorIdList = patchResult
.filter((item) => item.type === 'delete' || item.type === 'update')
.map((item) => item.index.dataId)
.filter(Boolean) as string[];
// 4. Update mongo updateTime(便于脏数据检查器识别)
const updateTime = mongoData.updateTime;
mongoData.updateTime = new Date();
await mongoData.save();
// 5. insert vector
const insertItems = patchResult.filter(
(item) => item.type === 'create' || item.type === 'update'
);
const tokens = await (async () => {
if (insertItems.length > 0) {
// Batch insert vectors
const result = await insertDatasetDataVector({
inputs: insertItems.map((item) => item.index.text),
model: getEmbeddingModel(model),
teamId: mongoData.teamId,
datasetId: mongoData.datasetId,
collectionId: mongoData.collectionId
});
// Update dataIds for the items
insertItems.forEach((item, index) => {
item.index.dataId = result.insertIds[index];
});
return result.tokens;
}
return 0;
})();
const newIndexes = patchResult
.filter((item) => item.type !== 'delete')
.map((item) => item.index) as DatasetDataIndexItemType[];
// 6. update mongo data
await mongoSessionRun(async (session) => {
// Update history
mongoData.history =
q !== mongoData.q || a !== mongoData.a
? [
{
q: mongoData.q,
a: mongoData.a,
updateTime: updateTime
},
...(mongoData.history?.slice(0, 9) || [])
]
: mongoData.history;
mongoData.q = q || mongoData.q;
mongoData.a = a ?? mongoData.a;
mongoData.indexes = newIndexes;
await mongoData.save({ session });
// update mongo data text
await MongoDatasetDataText.updateOne(
{ dataId: mongoData._id },
{ fullTextToken: await jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }) },
{ session }
);
// Delete vector
if (deleteVectorIdList.length > 0) {
await deleteDatasetDataVector({
teamId: mongoData.teamId,
idList: deleteVectorIdList
});
}
});
// Trigger collection update (async, with 5s delay and debounce)
pushCollectionUpdateJob({
collectionId: String(mongoData.collectionId),
datasetId: String(mongoData.datasetId),
teamId: String(mongoData.teamId)
});
return {
tokens
};
}
export const deleteDatasetData = async (data: DatasetDataItemType) => {
await mongoSessionRun(async (session) => {
if (data.imageId && !isS3ObjectKey(data.imageId, 'dataset')) {
return Promise.reject('Invalid dataset image key');
}
// 1. Delete MongoDB data
await MongoDatasetData.deleteOne({ _id: data.id }, { session });
await MongoDatasetDataText.deleteMany({ dataId: data.id }, { session });
if (data.imageId) {
await getS3DatasetSource().deleteDatasetFileByKey(data.imageId);
}
// 2. Delete vector data
await deleteDatasetDataVector({
teamId: data.teamId,
idList: data.indexes.map((item) => item.dataId)
});
});
// Trigger collection update (async, with 5s delay and debounce)
pushCollectionUpdateJob({
collectionId: String(data.collectionId),
datasetId: String(data.datasetId),
teamId: String(data.teamId)
});
};
projects/app/src/service/core/dataset/data/data.ts 0 → 100644
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index';
import { pushCollectionUpdateJob } from '@fastgpt/service/core/dataset/collection/mq';
import type {
UpdateDatasetDataPropsType,
DatasetDataItemType,
CreateDatasetDataPropsType
} from '@fastgpt/global/core/dataset/type';
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { type ClientSession } from '@fastgpt/service/common/mongo';
import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { isS3ObjectKey, removeS3TTL } from '@fastgpt/service/common/s3/utils';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import { DatasetDataIndexOperation } from '@/service/core/dataset/data/dataIndex';
type UpdateDatasetDataByIndexesProps = Omit<UpdateDatasetDataPropsType, 'indexes'> & {
indexes: NonNullable<UpdateDatasetDataPropsType['indexes']>;
model: string;
indexSize?: number;
};
/**
* 数据条目的写操作入口。
*
* 这个类负责协调一条 dataset data 相关的多份存储：
* - MongoDatasetData：主数据、Q/A、indexes、历史记录
* - MongoDatasetDataText：全文检索 token
* - 向量库：每个 index 对应的向量记录
* - S3：图片数据的生命周期或删除
*
* 修改这些流程时要特别注意写入顺序，避免 Mongo 中的 index dataId 和向量库记录不一致。
*/
export class DatasetDataOperation {
private readonly indexOperation: DatasetDataIndexOperation;
constructor(model?: string) {
this.indexOperation = new DatasetDataIndexOperation(model);
}
/**
* 通知 collection 重新计算统计信息和更新时间。
* data/index/vector 任一写操作完成后都应触发一次。
*/
private pushCollectionUpdate({
collectionId,
datasetId,
teamId
}: {
collectionId: string;
datasetId: string;
teamId: string;
}) {
pushCollectionUpdateJob({
collectionId: String(collectionId),
datasetId: String(datasetId),
teamId: String(teamId)
});
}
/**
* 创建一条 dataset data。
*
* 流程顺序：
* 1. 根据 Q/A 和传入 indexes 生成最终索引列表
* 2. 先写入向量库，拿到每条索引对应的 dataId
* 3. 写入主数据和全文检索 token
* 4. 如果图片来自 dataset S3 临时区，移除 TTL，避免被清理
*/
async create({
teamId,
tmbId,
datasetId,
collectionId,
q,
a,
imageId,
chunkIndex = 0,
indexSize = 512,
indexes,
indexPrefix,
embeddingModel,
imageDescMap,
session
}: CreateDatasetDataPropsType & {
embeddingModel: string;
indexSize?: number;
imageDescMap?: Record<string, string>;
session?: ClientSession;
}) {
if (!q || !datasetId || !collectionId || !embeddingModel) {
return Promise.reject('q, datasetId, collectionId, embeddingModel is required');
}
const embModel = getEmbeddingModel(embeddingModel);
indexSize = Math.min(embModel.maxToken, indexSize);
// 默认索引和自定义索引在这里统一规范化，确保后续向量写入的输入已去重、切分。
const newIndexes = await this.indexOperation.formatIndexes({
indexes,
q,
a,
indexSize,
maxIndexSize: embModel.maxToken,
indexPrefix
});
const { tokens, indexes: results } = await this.indexOperation.insertVectors({
indexes: newIndexes,
teamId,
datasetId,
collectionId
});
// 主数据保存的是带 dataId 的 indexes，因此需要先完成向量写入。
const [{ _id }] = await MongoDatasetData.create(
[
{
teamId,
tmbId,
datasetId,
collectionId,
q,
a,
imageId,
imageDescMap,
chunkIndex,
indexes: results
}
],
{ session, ordered: true }
);
// 单独维护分词后的全文检索内容，避免查询时临时分词。
await MongoDatasetDataText.create(
[
{
teamId,
datasetId,
collectionId,
dataId: _id,
fullTextToken: await jiebaSplit({ text: `${q}\n${a}`.trim() })
}
],
{ session, ordered: true }
);
// 图片在创建成功后从临时对象转为正式引用，不再允许 TTL 自动删除。
if (isS3ObjectKey(imageId, 'dataset')) {
await removeS3TTL({ key: imageId, bucketName: 'private', session });
}
this.pushCollectionUpdate({
collectionId,
datasetId,
teamId
});
return {
insertId: _id,
tokens
};
}
/**
* 按调用方传入的完整 indexes 更新数据。
*
* 这个路径用于“手动指定全部索引”的更新：调用方给出的 indexes 会和当前 indexes 做
* diff，新增/变更的索引重建向量，删除的索引清理旧向量。与 updateDefaultIndexes 不同，
* 它会以传入 indexes 为准更新整组索引。
*/
async updateByIndexes({
dataId,
q = '',
a,
indexes,
model,
indexSize = 512,
indexPrefix
}: UpdateDatasetDataByIndexesProps) {
if (!Array.isArray(indexes)) {
return Promise.reject('indexes is required');
}
const mongoData = await MongoDatasetData.findById(dataId);
if (!mongoData) return Promise.reject('Data not found');
const nextQ = q || mongoData.q || '';
const nextA = a ?? mongoData.a ?? '';
const formatIndexesResult = await this.indexOperation.formatIndexes({
indexes,
q: nextQ,
a: nextA,
indexSize,
maxIndexSize: getEmbeddingModel(model).maxToken,
indexPrefix
});
const indexesWithExistingDefaultIds = this.indexOperation.mergeExistingDefaultIndexIds({
currentIndexes: mongoData.indexes,
nextDefaultIndexes: formatIndexesResult
});
// patchResult 先保留旧 dataId；insertVectorForPatch 会为 create/update 项写入新向量并回填新 dataId。
const patchResult = this.indexOperation.buildPatch({
currentIndexes: mongoData.indexes,
nextIndexes: indexesWithExistingDefaultIds
});
const deleteVectorIdList = this.indexOperation.getDeleteVectorIdList(patchResult);
// 提前刷新 updateTime，保持旧接口“进入更新流程即更新时间”的行为。
const updateTime = mongoData.updateTime;
mongoData.updateTime = new Date();
await mongoData.save();
const tokens = await this.indexOperation.insertVectorForPatch({
patchResult,
teamId: mongoData.teamId,
datasetId: mongoData.datasetId,
collectionId: mongoData.collectionId
});
const newIndexes = this.indexOperation.getWritablePatchIndexes(patchResult);
await mongoSessionRun(async (session) => {
// 仅在 Q/A 变化时记录历史，最多保留最近 10 条旧内容。
mongoData.history =
nextQ !== mongoData.q || nextA !== mongoData.a
? [
{
q: mongoData.q,
a: mongoData.a,
updateTime
},
...(mongoData.history?.slice(0, 9) || [])
]
: mongoData.history;
mongoData.q = nextQ;
mongoData.a = nextA;
mongoData.indexes = newIndexes;
await mongoData.save({ session });
// Q/A 变化会影响全文检索结果，需要和主数据一并更新。
await MongoDatasetDataText.updateOne(
{ dataId: mongoData._id },
{ fullTextToken: await jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }) },
{ session }
);
// Mongo 已经指向新的 dataId 后再删旧向量，降低检索命中悬空向量 id 的风险。
await this.indexOperation.deleteVectors({
teamId: mongoData.teamId,
idList: deleteVectorIdList
});
});
this.pushCollectionUpdate({
collectionId: mongoData.collectionId,
datasetId: mongoData.datasetId,
teamId: mongoData.teamId
});
return {
tokens
};
}
/**
* 只根据 Q/A 重建默认索引，保留人工维护的自定义索引。
*
* 数据内容更新时会走这个路径：default 索引来自 Q/A，因此需要重新生成；自定义索引
* 是用户手动维护的检索提示，不应因为 Q/A 更新被覆盖。
*/
async updateDefaultIndexes({
dataId,
q = '',
a,
model,
indexSize = 512,
indexPrefix
}: {
dataId: string;
q: string;
a?: string;
model: string;
indexSize?: number;
indexPrefix?: string;
}) {
const mongoData = await MongoDatasetData.findById(dataId);
if (!mongoData) return Promise.reject('Data not found');
const embModel = getEmbeddingModel(model);
indexSize = Math.min(embModel.maxToken, indexSize);
const defaultIndexes = await this.indexOperation.getDefaultIndexes({
q,
a,
indexSize,
maxIndexSize: embModel.maxToken,
indexPrefix
});
// 默认索引文本没变化时复用旧 dataId，避免无意义的向量重建。
const nextDefaultIndexDrafts = this.indexOperation.mergeExistingDefaultIndexIds({
currentIndexes: mongoData.indexes,
nextDefaultIndexes: defaultIndexes
});
const patchResult = this.indexOperation.buildPatch({
currentIndexes: mongoData.indexes,
nextIndexes: nextDefaultIndexDrafts,
currentIndexFilter: (index) => index.type === DatasetDataIndexTypeEnum.default,
isSameIndex: (current, next) => current.text === next.text && current.type === next.type
});
const deleteVectorIdList = this.indexOperation.getDeleteVectorIdList(patchResult);
// 只为新增或变化的默认索引写入向量；未变化的索引继续使用原 dataId。
const tokens = await this.indexOperation.insertVectorForPatch({
patchResult,
teamId: mongoData.teamId,
datasetId: mongoData.datasetId,
collectionId: mongoData.collectionId
});
const nextDefaultIndexes = this.indexOperation.getWritablePatchIndexes(patchResult);
const updateTime = mongoData.updateTime;
const nextQ = q || mongoData.q;
const nextA = a ?? mongoData.a;
const isDataChanged = nextQ !== mongoData.q || nextA !== mongoData.a;
const updateFields = {
...(isDataChanged
? {
history: {
$literal: [
{
q: mongoData.q,
a: mongoData.a,
updateTime
},
...(mongoData.history?.slice(0, 9) || [])
]
}
}
: {}),
q: { $literal: nextQ },
a: { $literal: nextA },
indexes: {
$concatArrays: [
{
$filter: {
input: '$indexes',
as: 'index',
cond: { $ne: ['$$index.type', DatasetDataIndexTypeEnum.default] }
}
},
{ $literal: nextDefaultIndexes }
]
},
updateTime: { $literal: new Date() }
};
await mongoSessionRun(async (session) => {
// Only replace default indexes at write time. Custom indexes may be created concurrently.
await MongoDatasetData.updateOne(
{ _id: mongoData._id },
[
{
$set: updateFields
}
],
{ session }
);
// 默认索引来自 Q/A，全文检索 token 也必须和 Q/A 同步。
await MongoDatasetDataText.updateOne(
{ dataId: mongoData._id },
{ fullTextToken: await jiebaSplit({ text: `${nextQ}\n${nextA}`.trim() }) },
{ session }
);
// 等 Mongo indexes 更新完成后再删除旧向量，避免短时间内出现悬空引用。
await this.indexOperation.deleteVectors({
teamId: mongoData.teamId,
idList: deleteVectorIdList
});
});
this.pushCollectionUpdate({
collectionId: mongoData.collectionId,
datasetId: mongoData.datasetId,
teamId: mongoData.teamId
});
return {
tokens
};
}
/**
* 删除一条 dataset data 以及所有派生资源。
*
* 删除范围包含主数据、全文检索 token、图片文件和向量记录。图片 key 需要先校验，
* 防止误删非 dataset 来源的 S3 对象。
*/
async delete(data: DatasetDataItemType) {
await mongoSessionRun(async (session) => {
await MongoDatasetData.deleteOne({ _id: data.id }, { session });
await MongoDatasetDataText.deleteMany({ dataId: data.id }, { session });
// 主数据删除后清理图片对象，避免孤儿文件继续占用存储。
if (data.imageId && isS3ObjectKey(data.imageId, 'dataset')) {
await getS3DatasetSource().deleteDatasetFileByKey(data.imageId);
}
// data.indexes 中的 dataId 即向量 id，删除数据时需要全部清理。
await this.indexOperation.deleteVectors({
teamId: data.teamId,
idList: data.indexes.map((item) => item.dataId)
});
});
this.pushCollectionUpdate({
collectionId: data.collectionId,
datasetId: data.datasetId,
teamId: data.teamId
});
}
}
/**
* 创建 dataset data 的服务函数。
* API 层使用该函数完成数据、全文索引、向量和图片 TTL 的一次性写入。
*/
export const createDatasetData = async (
props: CreateDatasetDataPropsType & {
embeddingModel: string;
indexSize?: number;
imageDescMap?: Record<string, string>;
session?: ClientSession;
}
) => {
return new DatasetDataOperation(props.embeddingModel).create(props);
};
/**
* 按完整 indexes 更新 dataset data。
* 适用于调用方显式提交整组索引的场景。
*/
export const updateDatasetDataByIndexes = async (props: UpdateDatasetDataByIndexesProps) => {
return new DatasetDataOperation(props.model).updateByIndexes(props);
};
/**
* 根据 Q/A 更新默认索引，同时保留自定义索引。
* 适用于普通数据内容编辑场景。
*/
export const updateDatasetDataDefaultIndexes = async (props: {
dataId: string;
q: string;
a?: string;
model: string;
indexSize?: number;
indexPrefix?: string;
}) => {
return new DatasetDataOperation(props.model).updateDefaultIndexes(props);
};
/**
* 删除 dataset data 及其全文索引、图片和向量等派生资源。
*/
export const deleteDatasetData = async (data: DatasetDataItemType) => {
return new DatasetDataOperation().delete(data);
};
projects/app/src/service/core/dataset/data/dataIndex.ts 0 → 100644
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import {
deleteDatasetDataVector,
insertDatasetDataVector
} from '@fastgpt/service/common/vectorDB/controller';
import { pushCollectionUpdateJob } from '@fastgpt/service/core/dataset/collection/mq';
import type {
DatasetDataIndexItemType,
DatasetDataItemType
} from '@fastgpt/global/core/dataset/type';
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken';
import { text2Chunks } from '@fastgpt/service/worker/function';
import type { EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.schema';
export type DatasetDataIndexDraft = Omit<DatasetDataIndexItemType, 'dataId'> & {
dataId?: string;
};
/**
* 描述一组新的索引应该如何应用到已有索引。
*
* `dataId` 同时也是向量库里的向量 id。更新索引时，patch 会先保留旧的
* dataId，等新向量写入成功后再替换成新的向量 id，最后写回 MongoDB。
*/
export type DatasetDataIndexPatch =
| {
type: 'create';
index: DatasetDataIndexDraft;
}
| {
type: 'update';
index: DatasetDataIndexItemType;
}
| {
type: 'delete';
index: DatasetDataIndexItemType;
}
| {
type: 'unChange';
index: DatasetDataIndexItemType;
};
/**
* 给索引文本补充集合上下文，让检索时能利用前缀信息。
* 这里需要保持幂等，因为调用方可能传入已经在上一次全量更新中加过前缀的索引。
*/
const formatIndexTextWithPrefix = (text: string, indexPrefix?: string) => {
if (indexPrefix && !text.startsWith(indexPrefix)) {
return `${indexPrefix}\n${text}`;
}
return text;
};
/**
* 数据索引变更的共享操作类。
*
* 这里集中维护索引的完整生命周期：
* - 根据 Q/A 文本生成默认索引
* - 在向量化前规范化并切分自定义索引
* - 对比当前索引和下一版索引，生成 patch
* - 写入或删除向量记录
* - 更新 Mongo 中的 `indexes` 数组
*
* 这些步骤需要放在一起维护，因为 Mongo 索引里的 `dataId` 必须和向量库 id 保持一致。
*/
export class DatasetDataIndexOperation {
private readonly model?: string | EmbeddingModelItemType;
constructor(model?: string | EmbeddingModelItemType) {
this.model = model;
}
get maxToken() {
return this.getEmbeddingModel().maxToken;
}
private getEmbeddingModel() {
return typeof this.model === 'string' ? getEmbeddingModel(this.model) : this.model!;
}
/**
* 根据数据的 question 和 answer 生成系统维护的默认索引。
*
* 默认索引由数据内容重新生成，不允许用户单独编辑。question 和 answer 都可能
* 被切成多个 chunk，确保每条向量输入不会超过 embedding 模型限制。
*/
async getDefaultIndexes({
q = '',
a,
indexSize,
maxIndexSize,
indexPrefix
}: {
q?: string;
a?: string;
indexSize: number;
maxIndexSize?: number;
indexPrefix?: string;
}) {
const qChunks = (
await text2Chunks({
text: q,
chunkSize: indexSize,
maxSize: maxIndexSize ?? this.maxToken
})
).chunks;
const aChunks = a
? (
await text2Chunks({
text: a,
chunkSize: indexSize,
maxSize: maxIndexSize ?? this.maxToken
})
).chunks
: [];
return [
...qChunks.map((text) => ({
text: formatIndexTextWithPrefix(text, indexPrefix),
type: DatasetDataIndexTypeEnum.default
})),
...aChunks.map((text) => ({
text: formatIndexTextWithPrefix(text, indexPrefix),
type: DatasetDataIndexTypeEnum.default
}))
];
}
/**
* 在对比或写入向量前，规范化所有索引草稿。
*
* 该流程会保留自定义索引、根据 Q/A 重新生成默认索引、按文本去重，并切分过长的
* 自定义索引。文本未变化时会沿用已有 dataId，避免重复重建向量。
*/
async formatIndexes({
indexes = [],
q,
a = '',
indexSize,
maxIndexSize,
indexPrefix
}: {
indexes?: DatasetDataIndexDraft[];
q: string;
a?: string;
indexSize: number;
maxIndexSize?: number;
indexPrefix?: string;
}): Promise<DatasetDataIndexDraft[]> {
indexes = indexes.map((item) => ({
text: typeof item.text === 'string' ? item.text : String(item.text),
type: item.type || DatasetDataIndexTypeEnum.custom,
dataId: item.dataId
}));
const defaultIndexes = await this.getDefaultIndexes({
q,
a,
indexSize,
maxIndexSize: maxIndexSize ?? this.maxToken,
indexPrefix
});
// 把 dataId 合并到旧的 index 上，避免重复生成
const concatDefaultIndexes = defaultIndexes.map((item) => {
const oldIndex = indexes.find((index) => index.text === item.text);
if (oldIndex) {
return {
type: DatasetDataIndexTypeEnum.default,
text: item.text,
dataId: oldIndex.dataId
};
}
return item;
});
// 筛选掉重复索引：不是默认的，文案相同的
indexes = indexes.filter(
(item, index, self) =>
item.type !== DatasetDataIndexTypeEnum.default &&
!concatDefaultIndexes.find((t) => t.text === item.text) &&
index === self.findIndex((t) => t.text === item.text)
);
indexes.push(...concatDefaultIndexes);
const checkedIndexes = (
await Promise.all(
indexes.map(async (item) => {
if (item.type === DatasetDataIndexTypeEnum.default) {
return item;
}
const tokens = await countPromptTokens(item.text);
if (tokens > (maxIndexSize ?? this.maxToken)) {
const splitText = (
await text2Chunks({
text: item.text,
chunkSize: indexSize,
maxSize: maxIndexSize ?? this.maxToken
})
).chunks;
return splitText.map((text) => ({
text,
type: item.type
}));
}
return item;
})
)
)
.flat()
.filter((item) => !!item.text.trim());
return indexPrefix
? checkedIndexes.map((index) => {
if (index.type === DatasetDataIndexTypeEnum.custom) return index;
return {
...index,
text: formatIndexTextWithPrefix(index.text, indexPrefix)
};
})
: checkedIndexes;
}
/**
* 默认索引重新生成后，重新挂回当前默认索引的 dataId。
*
* 默认索引来自内容，调用方通常会先生成没有 id 的草稿。这里按文本匹配，让未变化的
* 默认索引继续指向已有向量记录。
*/
mergeExistingDefaultIndexIds({
currentIndexes,
nextDefaultIndexes
}: {
currentIndexes: DatasetDataIndexItemType[];
nextDefaultIndexes: DatasetDataIndexDraft[];
}) {
const existingDefaultMap = new Map(
currentIndexes
.filter((index) => index.type === DatasetDataIndexTypeEnum.default)
.map((index) => [index.text, index])
);
return nextDefaultIndexes.map((index) => {
const existingIndex = existingDefaultMap.get(index.text);
return {
...index,
...(existingIndex?.dataId && { dataId: existingIndex.dataId })
};
});
}
/**
* 对比已存索引和下一版索引，生成需要执行的向量操作。
*
* 可选 filter 用于数据级更新时只 patch 默认索引，避免影响人工维护的自定义索引。
* 可选 comparator 用于让调用方决定除文本外的字段变化是否需要重建向量。
*/
buildPatch({
currentIndexes,
nextIndexes,
currentIndexFilter,
isSameIndex
}: {
currentIndexes: DatasetDataIndexItemType[];
nextIndexes: DatasetDataIndexDraft[];
currentIndexFilter?: (index: DatasetDataIndexItemType) => boolean;
isSameIndex?: (current: DatasetDataIndexItemType, next: DatasetDataIndexDraft) => boolean;
}) {
const patchResult: DatasetDataIndexPatch[] = [];
const filteredCurrentIndexes = currentIndexFilter
? currentIndexes.filter(currentIndexFilter)
: currentIndexes;
// 当前过滤集合中存在、但下一版不存在的索引，需要同时删除 Mongo 索引项和向量记录。
for (const item of filteredCurrentIndexes) {
const index = nextIndexes.find((index) => index.dataId === item.dataId);
if (!index) {
patchResult.push({
type: 'delete',
index: item
});
}
}
// 没有 dataId 的草稿需要创建新向量；已有 dataId 的草稿根据内容变化决定复用或重建。
for (const item of nextIndexes) {
if (!item.dataId) {
patchResult.push({
type: 'create',
index: item
});
continue;
}
const index = currentIndexes.find((index) => index.dataId === item.dataId);
if (!index) continue;
if ((isSameIndex ?? ((current, next) => current.text === next.text))(index, item)) {
patchResult.push({
type: 'unChange',
index: {
...item,
dataId: index.dataId
}
});
} else {
patchResult.push({
type: 'update',
index: {
...item,
dataId: index.dataId
}
});
}
}
return patchResult;
}
/**
* 收集应用 patch 后会失效的向量 id。
*
* update 也需要删除旧向量，因为新向量写入后，patch 中的 dataId 会被替换为新的
* 向量 id，再写回 Mongo。
*/
getDeleteVectorIdList(patchResult: DatasetDataIndexPatch[]) {
return patchResult
.filter((item) => item.type === 'delete' || item.type === 'update')
.map((item) => item.index.dataId)
.filter(Boolean) as string[];
}
/**
* 将 patch 转回 Mongo `indexes` 数组结构。
* delete 项会被过滤掉，create/update/unChange 项会保留下来。
*/
getWritablePatchIndexes(patchResult: DatasetDataIndexPatch[]) {
return patchResult
.filter((item) => item.type !== 'delete')
.map((item) => item.index) as DatasetDataIndexItemType[];
}
/**
* 为需要新向量 id 的 patch 项写入向量。
*
* 这里会原地修改 patch，让后续 Mongo 写入直接使用新的 dataId，避免调用方再做一次映射。
*/
async insertVectorForPatch({
patchResult,
teamId,
datasetId,
collectionId
}: {
patchResult: DatasetDataIndexPatch[];
teamId: string;
datasetId: string;
collectionId: string;
}) {
const insertItems = patchResult.filter(
(item) => item.type === 'create' || item.type === 'update'
);
if (insertItems.length === 0) return 0;
const result = await insertDatasetDataVector({
inputs: insertItems.map((item) => item.index.text),
model: this.getEmbeddingModel(),
teamId,
datasetId,
collectionId
});
insertItems.forEach((item, index) => {
item.index.dataId = result.insertIds[index];
});
return result.tokens;
}
/**
* 为一组新的索引草稿写入向量。
* 单个索引创建/更新路径没有 patch 对象，会使用这个方法。
*/
async insertVectors({
indexes,
teamId,
datasetId,
collectionId
}: {
indexes: DatasetDataIndexDraft[];
teamId: string;
datasetId: string;
collectionId: string;
}) {
const { tokens, insertIds } = await insertDatasetDataVector({
inputs: indexes.map((item) => item.text),
model: this.getEmbeddingModel(),
teamId,
datasetId,
collectionId
});
return {
tokens,
indexes: indexes.map((item, index) => ({
...item,
dataId: insertIds[index]
})) as DatasetDataIndexItemType[]
};
}
/**
* 按 id 删除向量。
* 调用方需要负责同步删除或替换 Mongo 中的索引项。
*/
async deleteVectors({ teamId, idList }: { teamId: string; idList: string[] }) {
if (idList.length === 0) return;
await deleteDatasetDataVector({
teamId,
idList
});
}
/**
* 创建或更新一条人工维护的数据索引。
*
* 系统维护的索引类型不允许单独保存，因为它们的生命周期绑定在数据内容或图片处理上。
* 更新时会先写入新向量，再替换 Mongo 索引项；旧向量会在索引项替换成功后删除。
*/
async writeDatasetDataIndex({
data,
indexDataId,
type,
text
}: {
data: DatasetDataItemType;
indexDataId?: string;
type: DatasetDataIndexTypeEnum;
text: string;
}) {
const trimText = text.trim();
if (!trimText) {
return Promise.reject('Dataset data index text is required');
}
if (type === DatasetDataIndexTypeEnum.default) {
return Promise.reject('System indexes cannot be saved separately');
}
const textTokens = await countPromptTokens(trimText);
if (textTokens > this.maxToken) {
return Promise.reject('Dataset data index text is too long');
}
const targetIndex = indexDataId
? data.indexes.find((item) => item.dataId === indexDataId)
: undefined;
// 有 indexDataId 但是找不到对应的 index，认为是错误的数据
if (indexDataId && !targetIndex) {
return Promise.reject('Dataset data index not found');
}
// 内容和类型都没变时直接复用旧索引，避免重复消耗 embedding tokens。
if (targetIndex && targetIndex.text === trimText && targetIndex.type === type) {
return {
index: targetIndex,
tokens: 0
};
}
const { indexes, tokens } = await this.insertVectors({
indexes: [
{
type,
text: trimText
}
],
teamId: data.teamId,
datasetId: data.datasetId,
collectionId: data.collectionId
});
const newIndex = indexes[0];
await mongoSessionRun(async (session) => {
if (targetIndex) {
// 先把 Mongo 索引替换成新的向量 id，再删除旧向量，避免检索指向不存在的向量记录。
await MongoDatasetData.updateOne(
{ _id: data.id, 'indexes.dataId': targetIndex.dataId },
{
$set: {
'indexes.$': newIndex,
updateTime: new Date()
}
},
{ session }
);
await this.deleteVectors({
teamId: data.teamId,
idList: [targetIndex.dataId]
});
} else {
// 人工添加的索引放在前面，后续读取时优先展示用户创建的检索提示。
await MongoDatasetData.updateOne(
{ _id: data.id },
{
$push: {
indexes: {
$each: [newIndex],
$position: 0
}
},
$set: {
updateTime: new Date()
}
},
{ session }
);
}
});
pushCollectionUpdateJob({
collectionId: String(data.collectionId),
datasetId: String(data.datasetId),
teamId: String(data.teamId)
});
return {
index: newIndex,
tokens
};
}
/**
* 删除一条人工维护的数据索引及其向量记录。
*/
async deleteDatasetDataIndex({
data,
indexDataId
}: {
data: DatasetDataItemType;
indexDataId: string;
}) {
const targetIndex = data.indexes.find((item) => item.dataId === indexDataId);
if (!targetIndex) {
return Promise.reject('Dataset data index not found');
}
if (targetIndex.type === DatasetDataIndexTypeEnum.default) {
return Promise.reject('System indexes cannot be deleted separately');
}
await mongoSessionRun(async (session) => {
await MongoDatasetData.updateOne(
{ _id: data.id },
{
$pull: {
indexes: { dataId: indexDataId }
},
$set: {
updateTime: new Date()
}
},
{ session }
);
await this.deleteVectors({
teamId: data.teamId,
idList: [indexDataId]
});
});
pushCollectionUpdateJob({
collectionId: String(data.collectionId),
datasetId: String(data.datasetId),
teamId: String(data.teamId)
});
}
}
/**
* 创建一条自定义数据索引及其向量记录。
*
* 底层操作会校验该索引不是系统维护类型，使用指定 embedding 模型写入新向量，
* 将生成的向量 dataId 追加到 data.indexes，并返回创建后的索引和 token 消耗。
*/
export const createDatasetDataIndex = async ({
data,
type,
text,
model
}: {
data: DatasetDataItemType;
type: DatasetDataIndexTypeEnum;
text: string;
model: string;
}) => {
return new DatasetDataIndexOperation(model).writeDatasetDataIndex({
data,
type,
text
});
};
/**
* 更新一条已有的自定义数据索引，并按需重建向量记录。
*
* 如果文本和类型都未变化，不会重新生成向量。否则会先写入替换向量，再替换 Mongo
* 中的索引项，最后删除旧向量，并返回更新后的索引和 token 消耗。
*/
export const updateDatasetDataIndex = async ({
data,
indexDataId,
type,
text,
model
}: {
data: DatasetDataItemType;
indexDataId: string;
type: DatasetDataIndexTypeEnum;
text: string;
model: string;
}) => {
return new DatasetDataIndexOperation(model).writeDatasetDataIndex({
data,
indexDataId,
type,
text
});
};
export const deleteDatasetDataIndex = async ({
data,
indexDataId
}: {
data: DatasetDataItemType;
indexDataId: string;
}) => {
return new DatasetDataIndexOperation().deleteDatasetDataIndex({
data,
indexDataId
});
};
projects/app/src/service/core/dataset/queues/generateVector.ts
import { insertData2Dataset } from '@/service/core/dataset/data/controller';
import { createDatasetData } from '@/service/core/dataset/data/data';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
......@@ -91,7 +91,7 @@ export async function generateVector(): Promise<any> {
return {
data
};
} catch (error) {
} catch {
return {
error: true
};
......@@ -238,7 +238,7 @@ const rebuildData = async ({ trainingData }: { trainingData: TrainingDataType })
}
})
);
} catch (error) {}
} catch {}
// update vector, update dataset_data rebuilding status, delete data from training
// 1. Insert new vector to dataset_data
......@@ -281,7 +281,7 @@ const rebuildData = async ({ trainingData }: { trainingData: TrainingDataType })
const insertData = async ({ trainingData }: { trainingData: TrainingDataType }) => {
return mongoSessionRun(async (session) => {
// insert new data to dataset
const { tokens } = await insertData2Dataset({
const { tokens } = await createDatasetData({
teamId: trainingData.teamId,
tmbId: trainingData.tmbId,
datasetId: trainingData.datasetId,
......
projects/app/src/web/core/dataset/api/data.ts
......@@ -6,7 +6,13 @@ import type {
GetQuoteDataBody as GetQuoteDataProps,
GetQuoteDataResponse,
InsertDataBody,
UpdateDatasetDataBody
CreateDatasetDataIndexBody,
DeleteDatasetDataIndexBody,
DeleteDatasetDataIndexResponse,
DatasetDataIndexResponse,
UpdateDatasetDataIndexBody,
UpdateDatasetDataBody,
UpdateDatasetDataResponse
} from '@fastgpt/global/openapi/core/dataset/data/api';
export const getDatasetDataList = (data: GetDatasetDataListProps) =>
......@@ -25,7 +31,13 @@ export const postInsertData2Dataset = (data: InsertDataBody) =>
* update one datasetData by id
*/
export const putDatasetDataById = (data: UpdateDatasetDataBody) =>
PUT('/core/dataset/data/update', data);
PUT<UpdateDatasetDataResponse>('/core/dataset/data/update', data);
export const createDatasetDataIndex = (data: CreateDatasetDataIndexBody) =>
POST<DatasetDataIndexResponse>('/core/dataset/data/index/create', data);
export const updateDatasetDataIndex = (data: UpdateDatasetDataIndexBody) =>
POST<DatasetDataIndexResponse>('/core/dataset/data/index/update', data);
/**
* 删除一条知识库数据
......@@ -33,6 +45,9 @@ export const putDatasetDataById = (data: UpdateDatasetDataBody) =>
export const delOneDatasetDataById = (id: string) =>
DELETE<string>(`/core/dataset/data/delete`, { id });
export const deleteDatasetDataIndex = (data: DeleteDatasetDataIndexBody) =>
POST<DeleteDatasetDataIndexResponse>('/core/dataset/data/index/delete', data);
// Get quote data
export const getQuoteData = (data: GetQuoteDataProps) =>
POST<GetQuoteDataResponse>(`/core/dataset/data/getQuoteData`, data);
projects/app/src/web/core/dataset/api/file.ts
import { POST } from '@/web/common/api/request';
import type {
GetSearchTestImagePreviewUrlsBody,
GetSearchTestImagePreviewUrlsResponse,
GetPreviewChunksBody,
GetPreviewChunksResponse,
PresignDatasetFilePostUrlBody
PresignDatasetFilePostUrlBody,
PresignSearchTestImageBody,
PresignSearchTestImageResponse
} from '@fastgpt/global/openapi/core/dataset/file/api';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
......@@ -14,3 +18,12 @@ export const getPreviewChunks = (data: GetPreviewChunksBody) =>
maxQuantity: 1,
timeout: 600000
});
export const getUploadSearchTestImagePresignedUrl = (data: PresignSearchTestImageBody) =>
POST<PresignSearchTestImageResponse>('/core/dataset/file/presignSearchTestImage', data);
export const postGetSearchTestImagePreviewUrls = (data: GetSearchTestImagePreviewUrlsBody) =>
POST<GetSearchTestImagePreviewUrlsResponse>(
'/core/dataset/file/getSearchTestImagePreviewUrls',
data
);
projects/app/src/web/core/dataset/store/searchTest.ts
......@@ -10,6 +10,10 @@ export type SearchTestStoreItemType = {
time: Date;
duration: string;
results: SearchDataResponseItemType[];
queryImageRefs?: {
key: string;
previewUrl?: string;
}[];
searchMode: `${DatasetSearchModeEnum}`;
limit: number;
usingReRank: boolean;
......@@ -27,7 +31,7 @@ type State = {
export const useSearchTestStore = create<State>()(
devtools(
persist(
immer((set, get) => ({
immer((set) => ({
datasetTestList: [],
pushDatasetTestItem(data) {
set((state) => {
......
projects/app/test/service/core/dataset/data/data.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { Types } from '@fastgpt/service/common/mongo';
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index';
import { MongoS3TTL } from '@fastgpt/service/common/s3/models/ttl';
import { S3Buckets } from '@fastgpt/service/common/s3/config/constants';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type {
DatasetDataIndexItemType,
DatasetDataItemType
} from '@fastgpt/global/core/dataset/type';
import { getRootUser } from '@test/datas/users';
import { createMockVectorsResponse, mockGetVectorsByText } from '@test/mocks/core/ai/embedding';
import { mockVectorDelete, mockVectorInsert, resetVectorMocks } from '@test/mocks/common/vector';
import {
createDatasetData,
deleteDatasetData,
updateDatasetDataByIndexes,
updateDatasetDataDefaultIndexes
} from '@/service/core/dataset/data/data';
const { mockDeleteDatasetFileByKey, mockCountPromptTokens } = vi.hoisted(() => ({
mockDeleteDatasetFileByKey: vi.fn(),
mockCountPromptTokens: vi.fn(async (text: string) => text.length)
}));
vi.mock('@fastgpt/service/common/s3/sources/dataset', () => ({
getS3DatasetSource: vi.fn(() => ({
deleteDatasetFileByKey: mockDeleteDatasetFileByKey
}))
}));
vi.mock('@fastgpt/service/common/string/tiktoken', () => ({
countPromptTokens: mockCountPromptTokens
}));
const embeddingModel = {
model: 'text-embedding-3-small',
name: 'text-embedding-3-small',
maxToken: 100
} as any;
const createDatasetContext = async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test dataset',
teamId: root.teamId,
tmbId: root.tmbId,
type: DatasetTypeEnum.dataset,
vectorModel: 'text-embedding-3-small',
agentModel: 'gpt-4o-mini'
});
const collection = await MongoDatasetCollection.create({
name: 'test collection',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
return { root, dataset, collection };
};
const createMongoData = async ({
q = 'old question',
a = 'old answer',
indexes,
history
}: {
q?: string;
a?: string;
indexes?: DatasetDataIndexItemType[];
history?: DatasetDataItemType['history'];
} = {}) => {
const { root, dataset, collection } = await createDatasetContext();
const data = await MongoDatasetData.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
q,
a,
history,
indexes: indexes ?? [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'old custom index',
dataId: 'custom_old'
},
{
type: DatasetDataIndexTypeEnum.default,
text: q,
dataId: 'default_old'
}
]
});
await MongoDatasetDataText.create({
teamId: root.teamId,
datasetId: dataset._id,
collectionId: collection._id,
dataId: data._id,
fullTextToken: 'old token'
});
return { root, dataset, collection, data };
};
const toDataItem = (
data: Awaited<ReturnType<typeof MongoDatasetData.create>>
): DatasetDataItemType =>
({
id: String(data._id),
teamId: String(data.teamId),
tmbId: String(data.tmbId),
datasetId: String(data.datasetId),
collectionId: String(data.collectionId),
q: data.q,
a: data.a,
imageId: data.imageId,
chunkIndex: data.chunkIndex,
updateTime: data.updateTime,
history: data.history,
indexes: data.indexes.map((index) => ({
type: index.type,
text: index.text,
dataId: index.dataId
}))
}) as DatasetDataItemType;
describe('Dataset data service', () => {
beforeEach(() => {
resetVectorMocks();
mockGetVectorsByText.mockClear();
mockDeleteDatasetFileByKey.mockReset();
mockCountPromptTokens.mockClear();
vi.mocked(getEmbeddingModel).mockReturnValue(embeddingModel);
mockGetVectorsByText.mockImplementation(async ({ input }) =>
createMockVectorsResponse(Array.isArray(input) ? input : [input])
);
mockVectorInsert.mockResolvedValue({
insertIds: ['id_1', 'id_2', 'id_3', 'id_4', 'id_5', 'id_6']
});
mockVectorDelete.mockResolvedValue(undefined);
});
describe('createDatasetData', () => {
it('should create data, full-text tokens, indexes and remove dataset image ttl', async () => {
const { root, dataset, collection } = await createDatasetContext();
const imageId = `dataset/${dataset._id}/image.png`;
await MongoS3TTL.create({
minioKey: imageId,
bucketName: S3Buckets.private,
expiredTime: new Date(Date.now() + 60_000)
});
const result = await createDatasetData({
teamId: String(root.teamId),
tmbId: String(root.tmbId),
datasetId: String(dataset._id),
collectionId: String(collection._id),
q: 'question',
a: 'answer',
imageId,
imageDescMap: { [imageId]: 'image desc' },
chunkIndex: 2,
indexes: [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'manual index'
}
],
embeddingModel: 'text-embedding-3-small',
indexSize: 50,
indexPrefix: 'prefix'
});
const data = await MongoDatasetData.findById(result.insertId).lean();
const dataText = await MongoDatasetDataText.findOne({ dataId: result.insertId }).lean();
const ttl = await MongoS3TTL.findOne({ minioKey: imageId }).lean();
expect(result.tokens).toBeGreaterThan(0);
expect(data).toEqual(
expect.objectContaining({
q: 'question',
a: 'answer',
imageId,
chunkIndex: 2,
imageDescMap: { [imageId]: 'image desc' }
})
);
expect(data?.indexes).toEqual(
expect.arrayContaining([
expect.objectContaining({
type: DatasetDataIndexTypeEnum.custom,
text: 'manual index',
dataId: 'id_1'
}),
expect.objectContaining({
type: DatasetDataIndexTypeEnum.default,
text: 'prefix\nquestion'
}),
expect.objectContaining({
type: DatasetDataIndexTypeEnum.default,
text: 'prefix\nanswer'
})
])
);
expect(dataText?.fullTextToken).toContain('question');
expect(dataText?.fullTextToken).toContain('answer');
expect(ttl).toBeNull();
});
it('should reject when required fields are missing', async () => {
const { root, dataset, collection } = await createDatasetContext();
await expect(
createDatasetData({
teamId: String(root.teamId),
tmbId: String(root.tmbId),
datasetId: String(dataset._id),
collectionId: String(collection._id),
q: '',
embeddingModel: 'text-embedding-3-small'
} as any)
).rejects.toBe('q, datasetId, collectionId, embeddingModel is required');
});
});
describe('updateDatasetDataByIndexes', () => {
it('should update q/a, replace full indexes, record history and delete stale vectors', async () => {
const { data } = await createMongoData({
q: 'old question',
a: 'old answer',
indexes: [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'old custom index',
dataId: 'custom_old'
},
{
type: DatasetDataIndexTypeEnum.default,
text: 'old question',
dataId: 'default_old'
},
{
type: DatasetDataIndexTypeEnum.custom,
text: 'remove me',
dataId: 'remove_old'
}
]
});
const oldUpdateTime = data.updateTime;
const result = await updateDatasetDataByIndexes({
dataId: String(data._id),
q: 'new question',
a: 'new answer',
indexes: [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'new custom index'
}
],
model: 'text-embedding-3-small',
indexSize: 50
});
const updatedData = await MongoDatasetData.findById(data._id).lean();
const updatedText = await MongoDatasetDataText.findOne({ dataId: data._id }).lean();
const expectedFullTextToken = await jiebaSplit({ text: 'new question\nnew answer' });
expect(result.tokens).toBeGreaterThan(0);
expect(updatedData?.q).toBe('new question');
expect(updatedData?.a).toBe('new answer');
expect(updatedData?.history?.[0]).toEqual(
expect.objectContaining({
q: 'old question',
a: 'old answer',
updateTime: oldUpdateTime
})
);
expect(updatedData?.indexes).toEqual(
expect.arrayContaining([
expect.objectContaining({
type: DatasetDataIndexTypeEnum.custom,
text: 'new custom index',
dataId: 'id_1'
}),
expect.objectContaining({
type: DatasetDataIndexTypeEnum.default,
text: 'new question'
}),
expect.objectContaining({
type: DatasetDataIndexTypeEnum.default,
text: 'new answer'
})
])
);
expect(updatedData?.indexes).toHaveLength(3);
expect(updatedText?.dataId.toString()).toBe(String(data._id));
expect(updatedText?.fullTextToken).toBe(expectedFullTextToken);
const deleteCall = mockVectorDelete.mock.calls[0]?.[0];
expect(String(deleteCall?.teamId)).toBe(String(data.teamId));
expect(deleteCall?.idList).toEqual(
expect.arrayContaining(['custom_old', 'default_old', 'remove_old'])
);
});
it('should reject invalid update-by-indexes requests', async () => {
const { data } = await createMongoData();
await expect(
updateDatasetDataByIndexes({
dataId: String(data._id),
q: 'question',
indexes: undefined as any,
model: 'text-embedding-3-small'
})
).rejects.toBe('indexes is required');
await expect(
updateDatasetDataByIndexes({
dataId: String(new Types.ObjectId()),
q: 'question',
indexes: [],
model: 'text-embedding-3-small'
})
).rejects.toBe('Data not found');
});
});
describe('updateDatasetDataDefaultIndexes', () => {
it('should replace only default indexes and keep concurrently added custom indexes', async () => {
const { data } = await createMongoData({
q: 'old question',
a: '',
indexes: [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'old custom index',
dataId: 'custom_old'
},
{
type: DatasetDataIndexTypeEnum.default,
text: 'old question',
dataId: 'default_old'
}
]
});
const updatePromise = updateDatasetDataDefaultIndexes({
dataId: String(data._id),
q: 'new question',
a: '',
model: 'text-embedding-3-small',
indexSize: 512
});
await MongoDatasetData.updateOne(
{ _id: data._id },
{
$push: {
indexes: {
$each: [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'concurrent custom index',
dataId: 'custom_concurrent'
}
],
$position: 0
}
}
}
);
await updatePromise;
const updatedData = await MongoDatasetData.findById(data._id).lean();
expect(updatedData?.q).toBe('new question');
expect(updatedData?.indexes).toEqual(
expect.arrayContaining([
expect.objectContaining({
type: DatasetDataIndexTypeEnum.custom,
text: 'old custom index',
dataId: 'custom_old'
}),
expect.objectContaining({
type: DatasetDataIndexTypeEnum.custom,
text: 'concurrent custom index',
dataId: 'custom_concurrent'
}),
expect.objectContaining({
type: DatasetDataIndexTypeEnum.default,
text: 'new question'
})
])
);
expect(
updatedData?.indexes.filter((index) => index.type === DatasetDataIndexTypeEnum.default)
).toHaveLength(1);
});
it('should keep history unchanged when q and a do not change', async () => {
const history = [
{
q: 'previous question',
a: 'previous answer',
updateTime: new Date('2024-01-01T00:00:00.000Z')
}
];
const { data } = await createMongoData({
q: 'same question',
a: 'same answer',
history,
indexes: [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'custom index',
dataId: 'custom_old'
},
{
type: DatasetDataIndexTypeEnum.default,
text: 'same question',
dataId: 'default_q'
},
{
type: DatasetDataIndexTypeEnum.default,
text: 'same answer',
dataId: 'default_a'
}
]
});
const result = await updateDatasetDataDefaultIndexes({
dataId: String(data._id),
q: 'same question',
a: 'same answer',
model: 'text-embedding-3-small',
indexSize: 50
});
const updatedData = await MongoDatasetData.findById(data._id).lean();
expect(result.tokens).toBe(0);
expect(updatedData?.history).toEqual([expect.objectContaining(history[0])]);
expect(mockVectorInsert).not.toHaveBeenCalled();
expect(mockVectorDelete).not.toHaveBeenCalled();
});
it('should reject when data does not exist', async () => {
await expect(
updateDatasetDataDefaultIndexes({
dataId: String(new Types.ObjectId()),
q: 'question',
model: 'text-embedding-3-small'
})
).rejects.toBe('Data not found');
});
});
describe('deleteDatasetData', () => {
it('should delete data, full-text data, dataset image and vectors', async () => {
const { data } = await createMongoData({
indexes: [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'custom',
dataId: 'custom_id'
},
{
type: DatasetDataIndexTypeEnum.default,
text: 'default',
dataId: 'default_id'
}
]
});
data.imageId = `dataset/${data.datasetId}/image.png`;
await data.save();
const dataItem = toDataItem(data);
await deleteDatasetData(dataItem);
expect(await MongoDatasetData.findById(data._id).lean()).toBeNull();
expect(await MongoDatasetDataText.findOne({ dataId: data._id }).lean()).toBeNull();
expect(mockDeleteDatasetFileByKey).toHaveBeenCalledWith(data.imageId);
expect(mockVectorDelete).toHaveBeenCalledWith({
teamId: String(data.teamId),
idList: ['custom_id', 'default_id']
});
});
it('should skip image deletion and vector deletion for non-dataset image and empty indexes', async () => {
const { data } = await createMongoData({ indexes: [] });
data.imageId = 'chat/app/file.png';
await data.save();
await deleteDatasetData(toDataItem(data));
expect(mockDeleteDatasetFileByKey).not.toHaveBeenCalled();
expect(mockVectorDelete).not.toHaveBeenCalled();
});
});
});
projects/app/test/service/core/dataset/data/dataIndex.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { Types } from '@fastgpt/service/common/mongo';
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type {
DatasetDataIndexItemType,
DatasetDataItemType
} from '@fastgpt/global/core/dataset/type';
import { getRootUser } from '@test/datas/users';
import { mockGetVectorsByText, createMockVectorsResponse } from '@test/mocks/core/ai/embedding';
import { mockVectorDelete, mockVectorInsert, resetVectorMocks } from '@test/mocks/common/vector';
import {
createDatasetDataIndex,
DatasetDataIndexOperation,
deleteDatasetDataIndex,
updateDatasetDataIndex
} from '@/service/core/dataset/data/dataIndex';
const { mockCountPromptTokens } = vi.hoisted(() => ({
mockCountPromptTokens: vi.fn(async (text: string) => text.length)
}));
vi.mock('@fastgpt/service/common/string/tiktoken', () => ({
countPromptTokens: mockCountPromptTokens
}));
const embeddingModel = {
model: 'text-embedding-3-small',
name: 'text-embedding-3-small',
maxToken: 12
} as any;
const createDatasetContext = async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test dataset',
teamId: root.teamId,
tmbId: root.tmbId,
type: DatasetTypeEnum.dataset,
vectorModel: 'text-embedding-3-small',
agentModel: 'gpt-4o-mini'
});
const collection = await MongoDatasetCollection.create({
name: 'test collection',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
return { root, dataset, collection };
};
const createData = async (
indexes: DatasetDataIndexItemType[] = [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'old custom',
dataId: 'custom_old'
}
]
) => {
const { root, dataset, collection } = await createDatasetContext();
const data = await MongoDatasetData.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
q: 'question',
a: 'answer',
indexes
});
return {
root,
dataset,
collection,
data,
dataItem: {
id: String(data._id),
teamId: String(root.teamId),
tmbId: String(root.tmbId),
datasetId: String(dataset._id),
collectionId: String(collection._id),
q: data.q,
a: data.a,
chunkIndex: data.chunkIndex,
updateTime: data.updateTime,
indexes: data.indexes.map((index) => ({
type: index.type,
text: index.text,
dataId: index.dataId
}))
} as DatasetDataItemType
};
};
describe('DatasetDataIndexOperation', () => {
beforeEach(() => {
resetVectorMocks();
mockGetVectorsByText.mockClear();
mockCountPromptTokens.mockClear();
vi.mocked(getEmbeddingModel).mockReturnValue(embeddingModel);
mockGetVectorsByText.mockImplementation(async ({ input }) =>
createMockVectorsResponse(Array.isArray(input) ? input : [input])
);
mockVectorInsert.mockResolvedValue({
insertIds: ['id_1', 'id_2', 'id_3', 'id_4', 'id_5']
});
mockVectorDelete.mockResolvedValue(undefined);
});
describe('getDefaultIndexes', () => {
it('should create prefixed default indexes from question and answer', async () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
const result = await operation.getDefaultIndexes({
q: 'question text',
a: 'answer text',
indexSize: 50,
indexPrefix: 'collection title'
});
expect(result).toEqual([
{
type: DatasetDataIndexTypeEnum.default,
text: 'collection title\nquestion text'
},
{
type: DatasetDataIndexTypeEnum.default,
text: 'collection title\nanswer text'
}
]);
});
});
describe('formatIndexes', () => {
it('should normalize indexes, remove duplicate custom text and keep default indexes', async () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
const result = await operation.formatIndexes({
q: 'question',
a: '',
indexSize: 20,
indexes: [
{ type: DatasetDataIndexTypeEnum.custom, text: 'manual', dataId: 'manual_1' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'manual', dataId: 'manual_2' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'question', dataId: 'same_as_default' },
{ text: 123 as any }
]
});
expect(result).toEqual([
{
type: DatasetDataIndexTypeEnum.custom,
text: 'manual',
dataId: 'manual_1'
},
{
type: DatasetDataIndexTypeEnum.custom,
text: '123',
dataId: undefined
},
{
type: DatasetDataIndexTypeEnum.default,
text: 'question',
dataId: 'same_as_default'
}
]);
});
it('should split a custom index when token count exceeds max token', async () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
mockCountPromptTokens.mockResolvedValueOnce(30);
const result = await operation.formatIndexes({
q: '',
a: '',
indexSize: 8,
maxIndexSize: 12,
indexes: [
{
type: DatasetDataIndexTypeEnum.custom,
text: 'first sentence. second sentence. third sentence.'
}
]
});
expect(result.length).toBeGreaterThan(1);
expect(result.every((index) => index.type === DatasetDataIndexTypeEnum.custom)).toBe(true);
const mergedText = result.map((index) => index.text).join(' ');
expect(mergedText).toContain('first');
expect(mergedText).toContain('third');
});
});
describe('mergeExistingDefaultIndexIds', () => {
it('should reuse dataId for unchanged default indexes only', () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
const result = operation.mergeExistingDefaultIndexIds({
currentIndexes: [
{ type: DatasetDataIndexTypeEnum.default, text: 'same', dataId: 'default_old' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'same custom', dataId: 'custom_old' }
],
nextDefaultIndexes: [
{ type: DatasetDataIndexTypeEnum.default, text: 'same' },
{ type: DatasetDataIndexTypeEnum.default, text: 'new' }
]
});
expect(result).toEqual([
{ type: DatasetDataIndexTypeEnum.default, text: 'same', dataId: 'default_old' },
{ type: DatasetDataIndexTypeEnum.default, text: 'new' }
]);
});
});
describe('buildPatch', () => {
it('should build create, update, delete and unchanged patch items', () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
const result = operation.buildPatch({
currentIndexes: [
{ type: DatasetDataIndexTypeEnum.custom, text: 'keep', dataId: 'keep_id' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'old text', dataId: 'update_id' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'remove', dataId: 'delete_id' }
],
nextIndexes: [
{ type: DatasetDataIndexTypeEnum.custom, text: 'keep', dataId: 'keep_id' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'new text', dataId: 'update_id' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'create' }
]
});
expect(result.map((item) => item.type)).toEqual(['delete', 'unChange', 'update', 'create']);
expect(operation.getDeleteVectorIdList(result)).toEqual(['delete_id', 'update_id']);
expect(operation.getWritablePatchIndexes(result)).toEqual([
{ type: DatasetDataIndexTypeEnum.custom, text: 'keep', dataId: 'keep_id' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'new text', dataId: 'update_id' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'create' }
]);
});
it('should allow patching only filtered current indexes', () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
const result = operation.buildPatch({
currentIndexes: [
{ type: DatasetDataIndexTypeEnum.custom, text: 'custom', dataId: 'custom_id' },
{ type: DatasetDataIndexTypeEnum.default, text: 'old default', dataId: 'default_id' }
],
nextIndexes: [{ type: DatasetDataIndexTypeEnum.default, text: 'new default' }],
currentIndexFilter: (index) => index.type === DatasetDataIndexTypeEnum.default
});
expect(result).toEqual([
{
type: 'delete',
index: {
type: DatasetDataIndexTypeEnum.default,
text: 'old default',
dataId: 'default_id'
}
},
{
type: 'create',
index: {
type: DatasetDataIndexTypeEnum.default,
text: 'new default'
}
}
]);
});
});
describe('insertVectorForPatch', () => {
it('should insert vectors for create and update patch items and mutate dataId', async () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
const patchResult = operation.buildPatch({
currentIndexes: [
{ type: DatasetDataIndexTypeEnum.custom, text: 'old text', dataId: 'old_id' }
],
nextIndexes: [
{ type: DatasetDataIndexTypeEnum.custom, text: 'new text', dataId: 'old_id' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'new custom' }
]
});
const tokens = await operation.insertVectorForPatch({
patchResult,
teamId: 'team_id',
datasetId: 'dataset_id',
collectionId: 'collection_id'
});
expect(tokens).toBeGreaterThan(0);
expect(mockVectorInsert).toHaveBeenCalledTimes(1);
expect(operation.getWritablePatchIndexes(patchResult)).toEqual([
{ type: DatasetDataIndexTypeEnum.custom, text: 'new text', dataId: 'id_1' },
{ type: DatasetDataIndexTypeEnum.custom, text: 'new custom', dataId: 'id_2' }
]);
});
it('should return zero when no patch item needs vector insertion', async () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
const tokens = await operation.insertVectorForPatch({
patchResult: [
{
type: 'unChange',
index: { type: DatasetDataIndexTypeEnum.custom, text: 'same', dataId: 'same_id' }
}
],
teamId: 'team_id',
datasetId: 'dataset_id',
collectionId: 'collection_id'
});
expect(tokens).toBe(0);
expect(mockVectorInsert).not.toHaveBeenCalled();
});
});
describe('insertVectors and deleteVectors', () => {
it('should attach inserted vector ids to indexes', async () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
const result = await operation.insertVectors({
indexes: [
{ type: DatasetDataIndexTypeEnum.custom, text: 'one' },
{ type: DatasetDataIndexTypeEnum.default, text: 'two' }
],
teamId: 'team_id',
datasetId: 'dataset_id',
collectionId: 'collection_id'
});
expect(result.indexes).toEqual([
{ type: DatasetDataIndexTypeEnum.custom, text: 'one', dataId: 'id_1' },
{ type: DatasetDataIndexTypeEnum.default, text: 'two', dataId: 'id_2' }
]);
});
it('should skip vector delete when id list is empty', async () => {
const operation = new DatasetDataIndexOperation(embeddingModel);
await operation.deleteVectors({ teamId: 'team_id', idList: [] });
expect(mockVectorDelete).not.toHaveBeenCalled();
});
});
describe('writeDatasetDataIndex', () => {
it('should create a custom index and push it before existing indexes', async () => {
const { data, dataItem } = await createData();
const result = await createDatasetDataIndex({
data: dataItem,
type: DatasetDataIndexTypeEnum.custom,
text: ' new custom ',
model: 'text-embedding-3-small'
});
const updatedData = await MongoDatasetData.findById(data._id).lean();
expect(result.index).toEqual({
type: DatasetDataIndexTypeEnum.custom,
text: 'new custom',
dataId: 'id_1'
});
expect(result.tokens).toBeGreaterThan(0);
expect(updatedData?.indexes[0]).toEqual(
expect.objectContaining({
type: DatasetDataIndexTypeEnum.custom,
text: 'new custom',
dataId: 'id_1'
})
);
});
it('should update an existing custom index and delete the old vector', async () => {
const { data, dataItem } = await createData();
const result = await updateDatasetDataIndex({
data: dataItem,
indexDataId: 'custom_old',
type: DatasetDataIndexTypeEnum.custom,
text: 'updated',
model: 'text-embedding-3-small'
});
const updatedData = await MongoDatasetData.findById(data._id).lean();
expect(result.index).toEqual({
type: DatasetDataIndexTypeEnum.custom,
text: 'updated',
dataId: 'id_1'
});
expect(updatedData?.indexes).toEqual([
expect.objectContaining({
type: DatasetDataIndexTypeEnum.custom,
text: 'updated',
dataId: 'id_1'
})
]);
expect(mockVectorDelete).toHaveBeenCalledWith({
teamId: String(dataItem.teamId),
idList: ['custom_old']
});
});
it('should reuse existing index when text and type do not change', async () => {
const { dataItem } = await createData();
const result = await updateDatasetDataIndex({
data: dataItem,
indexDataId: 'custom_old',
type: DatasetDataIndexTypeEnum.custom,
text: 'old custom',
model: 'text-embedding-3-small'
});
expect(result).toEqual({
index: {
type: DatasetDataIndexTypeEnum.custom,
text: 'old custom',
dataId: 'custom_old'
},
tokens: 0
});
expect(mockVectorInsert).not.toHaveBeenCalled();
});
it('should reject invalid custom index save requests', async () => {
const { dataItem } = await createData();
await expect(
createDatasetDataIndex({
data: dataItem,
type: DatasetDataIndexTypeEnum.custom,
text: ' ',
model: 'text-embedding-3-small'
})
).rejects.toBe('Dataset data index text is required');
await expect(
createDatasetDataIndex({
data: dataItem,
type: DatasetDataIndexTypeEnum.default,
text: 'default',
model: 'text-embedding-3-small'
})
).rejects.toBe('System indexes cannot be saved separately');
await expect(
updateDatasetDataIndex({
data: dataItem,
indexDataId: 'missing_id',
type: DatasetDataIndexTypeEnum.custom,
text: 'valid',
model: 'text-embedding-3-small'
})
).rejects.toBe('Dataset data index not found');
});
it('should reject custom index text longer than model maxToken', async () => {
const { dataItem } = await createData();
mockCountPromptTokens.mockResolvedValueOnce(13);
await expect(
createDatasetDataIndex({
data: dataItem,
type: DatasetDataIndexTypeEnum.custom,
text: 'too long',
model: 'text-embedding-3-small'
})
).rejects.toBe('Dataset data index text is too long');
});
});
describe('deleteDatasetDataIndex', () => {
it('should delete a custom index and its vector', async () => {
const { data, dataItem } = await createData([
{ type: DatasetDataIndexTypeEnum.custom, text: 'custom', dataId: 'custom_id' },
{ type: DatasetDataIndexTypeEnum.default, text: 'default', dataId: 'default_id' }
]);
await deleteDatasetDataIndex({
data: dataItem,
indexDataId: 'custom_id'
});
const updatedData = await MongoDatasetData.findById(data._id).lean();
expect(updatedData?.indexes).toEqual([
expect.objectContaining({
type: DatasetDataIndexTypeEnum.default,
text: 'default',
dataId: 'default_id'
})
]);
expect(mockVectorDelete).toHaveBeenCalledWith({
teamId: String(dataItem.teamId),
idList: ['custom_id']
});
});
it('should reject deleting missing or system indexes', async () => {
const { dataItem } = await createData([
{ type: DatasetDataIndexTypeEnum.default, text: 'default', dataId: 'default_id' }
]);
await expect(
deleteDatasetDataIndex({
data: dataItem,
indexDataId: 'missing_id'
})
).rejects.toBe('Dataset data index not found');
await expect(
deleteDatasetDataIndex({
data: dataItem,
indexDataId: 'default_id'
})
).rejects.toBe('System indexes cannot be deleted separately');
});
});
describe('constructor', () => {
it('should use provided embedding model object for maxToken', () => {
const operation = new DatasetDataIndexOperation({
...embeddingModel,
maxToken: 321
});
expect(operation.maxToken).toBe(321);
});
it('should use the resolved embedding model when only a model name is provided', () => {
const operation = new DatasetDataIndexOperation('unknown-model');
expect(operation.maxToken).toBe(12);
});
it('keeps object id generation available for data fixtures', () => {
expect(Types.ObjectId.isValid(new Types.ObjectId())).toBe(true);
});
});
});
projects/app/test/service/core/dataset/data/utils.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getRootUser } from '@test/datas/users';
import { hasSameValue } from '@/service/core/dataset/data/utils';
const createDatasetContext = async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test dataset',
teamId: root.teamId,
tmbId: root.tmbId,
type: DatasetTypeEnum.dataset,
vectorModel: 'text-embedding-3-small',
agentModel: 'gpt-4o-mini'
});
const collection = await MongoDatasetCollection.create({
name: 'test collection',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
return { root, dataset, collection };
};
describe('hasSameValue', () => {
it('should resolve when no data has identical q and a in the same collection', async () => {
const { root, dataset, collection } = await createDatasetContext();
await MongoDatasetData.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
q: 'same question',
a: 'old answer',
indexes: []
});
await expect(
hasSameValue({
teamId: String(root.teamId),
datasetId: String(dataset._id),
collectionId: String(collection._id),
q: 'same question',
a: 'new answer'
})
).resolves.toBeUndefined();
});
it('should reject when identical q and a already exist', async () => {
const { root, dataset, collection } = await createDatasetContext();
await MongoDatasetData.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
q: 'same question',
a: '',
indexes: []
});
await expect(
hasSameValue({
teamId: String(root.teamId),
datasetId: String(dataset._id),
collectionId: String(collection._id),
q: 'same question'
})
).rejects.toBe('已经存在完全一致的数据');
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment